diff --git a/dc3d8690-0f9b-446a-b936-7da6b853de11-alerts.yaml b/dc3d8690-0f9b-446a-b936-7da6b853de11-alerts.yaml new file mode 100644 index 0000000000..1a3aa01244 --- /dev/null +++ b/dc3d8690-0f9b-446a-b936-7da6b853de11-alerts.yaml @@ -0,0 +1,37 @@ +name: dc3d8690-0f9b-446a-b936-7da6b853de11-alerts +interval: 1m +rules: + - record: slo:alert:ticket_window + expr: (max(slo:sli_error:ratio_rate5m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000347222 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate30m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000115741 * 5.000000000)) without (slo_window)) or (max(slo:sli_error:ratio_rate2h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000347222 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate6h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000115741 * 5.000000000)) without (slo_window)) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_severity: ticket + - record: slo:alert:page_window + expr: (max(slo:sli_error:ratio_rate5m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.001666667 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate30m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000694444 * 5.000000000)) without (slo_window)) or (max(slo:sli_error:ratio_rate2h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.001666667 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate6h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000694444 * 5.000000000)) without (slo_window)) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_severity: page + - alert: dc3d8690-0f9b-446a-b936-7da6b853de11-page-alerts + expr: (max(slo:sli_error:ratio_rate5m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.001666667 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate30m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000694444 * 5.000000000)) without (slo_window)) or (max(slo:sli_error:ratio_rate2h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.001666667 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate6h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000694444 * 5.000000000)) without (slo_window)) + labels: + opni_uuid: dc3d8690-0f9b-446a-b936-7da6b853de11-page-alerts + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_severity: page + annotations: + opni_uuid: dc3d8690-0f9b-446a-b936-7da6b853de11-page-alerts + - alert: dc3d8690-0f9b-446a-b936-7da6b853de11-ticket-alerts + expr: (max(slo:sli_error:ratio_rate5m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000347222 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate30m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000115741 * 5.000000000)) without (slo_window)) or (max(slo:sli_error:ratio_rate2h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000347222 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate6h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000115741 * 5.000000000)) without (slo_window)) + labels: + opni_uuid: dc3d8690-0f9b-446a-b936-7da6b853de11-ticket-alerts + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_severity: ticket + annotations: + opni_uuid: dc3d8690-0f9b-446a-b936-7da6b853de11-ticket-alerts diff --git a/dc3d8690-0f9b-446a-b936-7da6b853de11-metadata.yaml b/dc3d8690-0f9b-446a-b936-7da6b853de11-metadata.yaml new file mode 100644 index 0000000000..6336f7b6f1 --- /dev/null +++ b/dc3d8690-0f9b-446a-b936-7da6b853de11-metadata.yaml @@ -0,0 +1,49 @@ +name: dc3d8690-0f9b-446a-b936-7da6b853de11-metadata +interval: 1m +rules: + - record: slo:objective:ratio + expr: vector(0.950000000) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + - record: slo:error_budget:ratio + expr: vector(1-0.950000000) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + - record: slo:time_period:days + expr: vector(7) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + - record: slo:current_burn_rate:ratio + expr: |- + slo:sli_error:ratio_rate5m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} / on(slo_opni_id, slo_opni_service, slo_opni_name) group_left + slo:error_budget:ratio{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + - record: slo:period_burn_rate:ratio + expr: |- + slo:sli_error:ratio_rate7d{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} / on(slo_opni_id, slo_opni_service, slo_opni_name) group_left + slo:error_budget:ratio{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + - record: slo:period_error_budget_remaining:ratio + expr: 1 - slo:period_burn_rate:ratio{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + - record: opni_slo_info + expr: vector(1) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway diff --git a/dc3d8690-0f9b-446a-b936-7da6b853de11-recording.yaml b/dc3d8690-0f9b-446a-b936-7da6b853de11-recording.yaml new file mode 100644 index 0000000000..751353a494 --- /dev/null +++ b/dc3d8690-0f9b-446a-b936-7da6b853de11-recording.yaml @@ -0,0 +1,52 @@ +name: dc3d8690-0f9b-446a-b936-7da6b853de11-recording +interval: 1m +rules: + - record: slo:sli_error:ratio_rate5m + expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[5m])))/(sum(rate(process_max_fds{job="opni-gateway"}[5m])))) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_window: 5m + - record: slo:sli_error:ratio_rate30m + expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[30m])))/(sum(rate(process_max_fds{job="opni-gateway"}[30m])))) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_window: 30m + - record: slo:sli_error:ratio_rate1h + expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[1h])))/(sum(rate(process_max_fds{job="opni-gateway"}[1h])))) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_window: 1h + - record: slo:sli_error:ratio_rate2h + expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[2h])))/(sum(rate(process_max_fds{job="opni-gateway"}[2h])))) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_window: 2h + - record: slo:sli_error:ratio_rate6h + expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[6h])))/(sum(rate(process_max_fds{job="opni-gateway"}[6h])))) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_window: 6h + - record: slo:sli_error:ratio_rate1d + expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[1d])))/(sum(rate(process_max_fds{job="opni-gateway"}[1d])))) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_window: 1d + - record: slo:sli_error:ratio_rate7d + expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[7d])))/(sum(rate(process_max_fds{job="opni-gateway"}[7d])))) + labels: + slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11 + slo_opni_name: slo + slo_opni_service: opni-gateway + slo_window: 7d diff --git a/internal/cmd/testenv/main.go b/internal/cmd/testenv/main.go index 82a670f3dc..ef09bea240 100644 --- a/internal/cmd/testenv/main.go +++ b/internal/cmd/testenv/main.go @@ -48,7 +48,6 @@ import ( _ "github.com/rancher/opni/plugins/alerting/test" _ "github.com/rancher/opni/plugins/logging/test" _ "github.com/rancher/opni/plugins/metrics/test" - _ "github.com/rancher/opni/plugins/slo/test" ) func main() { @@ -190,11 +189,6 @@ func main() { c := make(chan os.Signal, 2) var closeOnce sync.Once signal.Notify(c, os.Interrupt) - iPort, _ = environment.StartInstrumentationServer() - kPort = environment.StartMockKubernetesMetricServer() - for i := 0; i < 100; i++ { - environment.SimulateKubeObject(kPort) - } testlog.Log.Infof(chalk.Green.Color("Instrumentation server listening on %d"), iPort) var client managementv1.ManagementClient if enableGateway { diff --git a/pkg/apis/core/v1/extensions.go b/pkg/apis/core/v1/extensions.go index aa9fd6b413..ce30f387f9 100644 --- a/pkg/apis/core/v1/extensions.go +++ b/pkg/apis/core/v1/extensions.go @@ -9,6 +9,15 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) +func (c *ClusterList) Has(clusterId string) bool { + for _, cluster := range c.Items { + if cluster.Id == clusterId { + return true + } + } + return false +} + func (s *HealthStatus) Summary() string { if s.Status == nil || s.Health == nil { return "Unknown" diff --git a/plugins/slo/apis/slo/extension.go b/pkg/apis/slo/v1/extensions.go similarity index 65% rename from plugins/slo/apis/slo/extension.go rename to pkg/apis/slo/v1/extensions.go index 44b95f7aee..363075745a 100644 --- a/plugins/slo/apis/slo/extension.go +++ b/pkg/apis/slo/v1/extensions.go @@ -1,4 +1,4 @@ -package slo +package v1 import ( "slices" @@ -19,5 +19,13 @@ func (s *ServiceList) ContainsId(id string) bool { for _, svc := range s.Items { svcIds = append(svcIds, svc.ServiceId) } - return !slices.Contains(svcIds, id) + return slices.Contains(svcIds, id) +} + +func (e *EventList) ContainsId(id string) bool { + var eventIds []string + for _, event := range e.Items { + eventIds = append(eventIds, event.Key) + } + return slices.Contains(eventIds, id) } diff --git a/plugins/slo/apis/slo/slo.pb.go b/pkg/apis/slo/v1/slo.pb.go similarity index 51% rename from plugins/slo/apis/slo/slo.pb.go rename to pkg/apis/slo/v1/slo.pb.go index 63f48f0b72..f6572ebd3e 100644 --- a/plugins/slo/apis/slo/slo.pb.go +++ b/pkg/apis/slo/v1/slo.pb.go @@ -2,9 +2,9 @@ // versions: // protoc-gen-go v1.30.0-devel // protoc v1.0.0 -// source: github.com/rancher/opni/plugins/slo/apis/slo/slo.proto +// source: github.com/rancher/opni/pkg/apis/slo/v1/slo.proto -package slo +package v1 import ( v11 "github.com/rancher/opni/pkg/apis/alerting/v1" @@ -76,11 +76,11 @@ func (x SLOStatusState) String() string { } func (SLOStatusState) Descriptor() protoreflect.EnumDescriptor { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_enumTypes[0].Descriptor() + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_enumTypes[0].Descriptor() } func (SLOStatusState) Type() protoreflect.EnumType { - return &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_enumTypes[0] + return &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_enumTypes[0] } func (x SLOStatusState) Number() protoreflect.EnumNumber { @@ -89,7 +89,54 @@ func (x SLOStatusState) Number() protoreflect.EnumNumber { // Deprecated: Use SLOStatusState.Descriptor instead. func (SLOStatusState) EnumDescriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{0} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{0} +} + +type DatasourceInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Items []string `protobuf:"bytes,1,rep,name=items,proto3" json:"items,omitempty"` +} + +func (x *DatasourceInfo) Reset() { + *x = DatasourceInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DatasourceInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DatasourceInfo) ProtoMessage() {} + +func (x *DatasourceInfo) ProtoReflect() protoreflect.Message { + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DatasourceInfo.ProtoReflect.Descriptor instead. +func (*DatasourceInfo) Descriptor() ([]byte, []int) { + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{0} +} + +func (x *DatasourceInfo) GetItems() []string { + if x != nil { + return x.Items + } + return nil } type MultiClusterSLO struct { @@ -104,7 +151,7 @@ type MultiClusterSLO struct { func (x *MultiClusterSLO) Reset() { *x = MultiClusterSLO{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[0] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -117,7 +164,7 @@ func (x *MultiClusterSLO) String() string { func (*MultiClusterSLO) ProtoMessage() {} func (x *MultiClusterSLO) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[0] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -130,7 +177,7 @@ func (x *MultiClusterSLO) ProtoReflect() protoreflect.Message { // Deprecated: Use MultiClusterSLO.ProtoReflect.Descriptor instead. func (*MultiClusterSLO) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{0} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{1} } func (x *MultiClusterSLO) GetCloneId() *v1.Reference { @@ -158,7 +205,7 @@ type MultiClusterFailures struct { func (x *MultiClusterFailures) Reset() { *x = MultiClusterFailures{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[1] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -171,7 +218,7 @@ func (x *MultiClusterFailures) String() string { func (*MultiClusterFailures) ProtoMessage() {} func (x *MultiClusterFailures) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[1] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -184,7 +231,7 @@ func (x *MultiClusterFailures) ProtoReflect() protoreflect.Message { // Deprecated: Use MultiClusterFailures.ProtoReflect.Descriptor instead. func (*MultiClusterFailures) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{1} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{2} } func (x *MultiClusterFailures) GetFailures() []string { @@ -208,7 +255,7 @@ type ListEventsRequest struct { func (x *ListEventsRequest) Reset() { *x = ListEventsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[2] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -221,7 +268,7 @@ func (x *ListEventsRequest) String() string { func (*ListEventsRequest) ProtoMessage() {} func (x *ListEventsRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[2] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -234,7 +281,7 @@ func (x *ListEventsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListEventsRequest.ProtoReflect.Descriptor instead. func (*ListEventsRequest) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{2} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{3} } func (x *ListEventsRequest) GetDatasource() string { @@ -278,7 +325,7 @@ type ListMetricsRequest struct { func (x *ListMetricsRequest) Reset() { *x = ListMetricsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[3] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -291,7 +338,7 @@ func (x *ListMetricsRequest) String() string { func (*ListMetricsRequest) ProtoMessage() {} func (x *ListMetricsRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[3] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -304,7 +351,7 @@ func (x *ListMetricsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListMetricsRequest.ProtoReflect.Descriptor instead. func (*ListMetricsRequest) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{3} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{4} } func (x *ListMetricsRequest) GetDatasource() string { @@ -340,7 +387,7 @@ type ListServicesRequest struct { func (x *ListServicesRequest) Reset() { *x = ListServicesRequest{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[4] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -353,7 +400,7 @@ func (x *ListServicesRequest) String() string { func (*ListServicesRequest) ProtoMessage() {} func (x *ListServicesRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[4] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -366,7 +413,7 @@ func (x *ListServicesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListServicesRequest.ProtoReflect.Descriptor instead. func (*ListServicesRequest) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{4} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{5} } func (x *ListServicesRequest) GetDatasource() string { @@ -394,7 +441,7 @@ type EventList struct { func (x *EventList) Reset() { *x = EventList{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[5] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -407,7 +454,7 @@ func (x *EventList) String() string { func (*EventList) ProtoMessage() {} func (x *EventList) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[5] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -420,7 +467,7 @@ func (x *EventList) ProtoReflect() protoreflect.Message { // Deprecated: Use EventList.ProtoReflect.Descriptor instead. func (*EventList) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{5} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{6} } func (x *EventList) GetItems() []*Event { @@ -442,7 +489,7 @@ type Event struct { func (x *Event) Reset() { *x = Event{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[6] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -455,7 +502,7 @@ func (x *Event) String() string { func (*Event) ProtoMessage() {} func (x *Event) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[6] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -468,7 +515,7 @@ func (x *Event) ProtoReflect() protoreflect.Message { // Deprecated: Use Event.ProtoReflect.Descriptor instead. func (*Event) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{6} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{7} } func (x *Event) GetKey() string { @@ -500,7 +547,7 @@ type ServiceInfo struct { func (x *ServiceInfo) Reset() { *x = ServiceInfo{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[7] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -513,7 +560,7 @@ func (x *ServiceInfo) String() string { func (*ServiceInfo) ProtoMessage() {} func (x *ServiceInfo) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[7] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -526,7 +573,7 @@ func (x *ServiceInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use ServiceInfo.ProtoReflect.Descriptor instead. func (*ServiceInfo) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{7} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{8} } func (x *ServiceInfo) GetJobId() string { @@ -576,7 +623,7 @@ type Service struct { func (x *Service) Reset() { *x = Service{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[8] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -589,7 +636,7 @@ func (x *Service) String() string { func (*Service) ProtoMessage() {} func (x *Service) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[8] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -602,7 +649,7 @@ func (x *Service) ProtoReflect() protoreflect.Message { // Deprecated: Use Service.ProtoReflect.Descriptor instead. func (*Service) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{8} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{9} } func (x *Service) GetServiceId() string { @@ -630,7 +677,7 @@ type ServiceList struct { func (x *ServiceList) Reset() { *x = ServiceList{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[9] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -643,7 +690,7 @@ func (x *ServiceList) String() string { func (*ServiceList) ProtoMessage() {} func (x *ServiceList) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[9] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -656,7 +703,7 @@ func (x *ServiceList) ProtoReflect() protoreflect.Message { // Deprecated: Use ServiceList.ProtoReflect.Descriptor instead. func (*ServiceList) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{9} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{10} } func (x *ServiceList) GetItems() []*Service { @@ -677,7 +724,7 @@ type Label struct { func (x *Label) Reset() { *x = Label{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[10] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -690,7 +737,7 @@ func (x *Label) String() string { func (*Label) ProtoMessage() {} func (x *Label) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[10] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -703,7 +750,7 @@ func (x *Label) ProtoReflect() protoreflect.Message { // Deprecated: Use Label.ProtoReflect.Descriptor instead. func (*Label) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{10} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{11} } func (x *Label) GetName() string { @@ -733,7 +780,7 @@ type Alert struct { func (x *Alert) Reset() { *x = Alert{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[11] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -746,7 +793,7 @@ func (x *Alert) String() string { func (*Alert) ProtoMessage() {} func (x *Alert) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[11] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -759,7 +806,7 @@ func (x *Alert) ProtoReflect() protoreflect.Message { // Deprecated: Use Alert.ProtoReflect.Descriptor instead. func (*Alert) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{11} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{12} } func (x *Alert) GetName() string { @@ -855,7 +902,7 @@ type ServiceLevelObjective struct { func (x *ServiceLevelObjective) Reset() { *x = ServiceLevelObjective{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[12] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -868,7 +915,7 @@ func (x *ServiceLevelObjective) String() string { func (*ServiceLevelObjective) ProtoMessage() {} func (x *ServiceLevelObjective) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[12] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -881,7 +928,7 @@ func (x *ServiceLevelObjective) ProtoReflect() protoreflect.Message { // Deprecated: Use ServiceLevelObjective.ProtoReflect.Descriptor instead. func (*ServiceLevelObjective) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{12} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{13} } func (x *ServiceLevelObjective) GetName() string { @@ -986,7 +1033,7 @@ type CreateSLORequest struct { func (x *CreateSLORequest) Reset() { *x = CreateSLORequest{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[13] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -999,7 +1046,7 @@ func (x *CreateSLORequest) String() string { func (*CreateSLORequest) ProtoMessage() {} func (x *CreateSLORequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[13] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1012,7 +1059,7 @@ func (x *CreateSLORequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateSLORequest.ProtoReflect.Descriptor instead. func (*CreateSLORequest) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{13} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{14} } func (x *CreateSLORequest) GetSlo() *ServiceLevelObjective { @@ -1033,7 +1080,7 @@ type EventPairList struct { func (x *EventPairList) Reset() { *x = EventPairList{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[14] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1046,7 +1093,7 @@ func (x *EventPairList) String() string { func (*EventPairList) ProtoMessage() {} func (x *EventPairList) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[14] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1059,7 +1106,7 @@ func (x *EventPairList) ProtoReflect() protoreflect.Message { // Deprecated: Use EventPairList.ProtoReflect.Descriptor instead. func (*EventPairList) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{14} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{15} } func (x *EventPairList) GetItems() []*EventPair { @@ -1081,7 +1128,7 @@ type EventPair struct { func (x *EventPair) Reset() { *x = EventPair{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[15] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1094,7 +1141,7 @@ func (x *EventPair) String() string { func (*EventPair) ProtoMessage() {} func (x *EventPair) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[15] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1107,7 +1154,7 @@ func (x *EventPair) ProtoReflect() protoreflect.Message { // Deprecated: Use EventPair.ProtoReflect.Descriptor instead. func (*EventPair) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{15} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{16} } func (x *EventPair) GetKey() string { @@ -1137,7 +1184,7 @@ type SLOData struct { func (x *SLOData) Reset() { *x = SLOData{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[16] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1150,7 +1197,7 @@ func (x *SLOData) String() string { func (*SLOData) ProtoMessage() {} func (x *SLOData) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[16] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1163,7 +1210,7 @@ func (x *SLOData) ProtoReflect() protoreflect.Message { // Deprecated: Use SLOData.ProtoReflect.Descriptor instead. func (*SLOData) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{16} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{17} } func (x *SLOData) GetId() string { @@ -1198,7 +1245,7 @@ type Target struct { func (x *Target) Reset() { *x = Target{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[17] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1211,7 +1258,7 @@ func (x *Target) String() string { func (*Target) ProtoMessage() {} func (x *Target) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[17] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1224,7 +1271,7 @@ func (x *Target) ProtoReflect() protoreflect.Message { // Deprecated: Use Target.ProtoReflect.Descriptor instead. func (*Target) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{17} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{18} } func (x *Target) GetValue() float64 { @@ -1246,7 +1293,7 @@ type Metric struct { func (x *Metric) Reset() { *x = Metric{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[18] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1259,7 +1306,7 @@ func (x *Metric) String() string { func (*Metric) ProtoMessage() {} func (x *Metric) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[18] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1272,7 +1319,7 @@ func (x *Metric) ProtoReflect() protoreflect.Message { // Deprecated: Use Metric.ProtoReflect.Descriptor instead. func (*Metric) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{18} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{19} } func (x *Metric) GetId() string { @@ -1302,7 +1349,7 @@ type MetricMetadata struct { func (x *MetricMetadata) Reset() { *x = MetricMetadata{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[19] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1315,7 +1362,7 @@ func (x *MetricMetadata) String() string { func (*MetricMetadata) ProtoMessage() {} func (x *MetricMetadata) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[19] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1328,7 +1375,7 @@ func (x *MetricMetadata) ProtoReflect() protoreflect.Message { // Deprecated: Use MetricMetadata.ProtoReflect.Descriptor instead. func (*MetricMetadata) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{19} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{20} } func (x *MetricMetadata) GetDescription() string { @@ -1363,7 +1410,7 @@ type MetricList struct { func (x *MetricList) Reset() { *x = MetricList{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[20] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1376,7 +1423,7 @@ func (x *MetricList) String() string { func (*MetricList) ProtoMessage() {} func (x *MetricList) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[20] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1389,7 +1436,7 @@ func (x *MetricList) ProtoReflect() protoreflect.Message { // Deprecated: Use MetricList.ProtoReflect.Descriptor instead. func (*MetricList) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{20} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{21} } func (x *MetricList) GetItems() []*Metric { @@ -1410,7 +1457,7 @@ type MetricGroupList struct { func (x *MetricGroupList) Reset() { *x = MetricGroupList{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[21] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1423,7 +1470,7 @@ func (x *MetricGroupList) String() string { func (*MetricGroupList) ProtoMessage() {} func (x *MetricGroupList) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[21] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1436,7 +1483,7 @@ func (x *MetricGroupList) ProtoReflect() protoreflect.Message { // Deprecated: Use MetricGroupList.ProtoReflect.Descriptor instead. func (*MetricGroupList) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{21} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{22} } func (x *MetricGroupList) GetGroupNameToMetrics() map[string]*MetricList { @@ -1457,7 +1504,7 @@ type ServiceLevelObjectiveList struct { func (x *ServiceLevelObjectiveList) Reset() { *x = ServiceLevelObjectiveList{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[22] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1470,7 +1517,7 @@ func (x *ServiceLevelObjectiveList) String() string { func (*ServiceLevelObjectiveList) ProtoMessage() {} func (x *ServiceLevelObjectiveList) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[22] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1483,7 +1530,7 @@ func (x *ServiceLevelObjectiveList) ProtoReflect() protoreflect.Message { // Deprecated: Use ServiceLevelObjectiveList.ProtoReflect.Descriptor instead. func (*ServiceLevelObjectiveList) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{22} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{23} } func (x *ServiceLevelObjectiveList) GetItems() []*SLOData { @@ -1504,7 +1551,7 @@ type SLOStatus struct { func (x *SLOStatus) Reset() { *x = SLOStatus{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[23] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1517,7 +1564,7 @@ func (x *SLOStatus) String() string { func (*SLOStatus) ProtoMessage() {} func (x *SLOStatus) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[23] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1530,7 +1577,7 @@ func (x *SLOStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use SLOStatus.ProtoReflect.Descriptor instead. func (*SLOStatus) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{23} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{24} } func (x *SLOStatus) GetState() SLOStatusState { @@ -1551,7 +1598,7 @@ type SLOPreviewResponse struct { func (x *SLOPreviewResponse) Reset() { *x = SLOPreviewResponse{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[24] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1564,7 +1611,7 @@ func (x *SLOPreviewResponse) String() string { func (*SLOPreviewResponse) ProtoMessage() {} func (x *SLOPreviewResponse) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[24] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1577,7 +1624,7 @@ func (x *SLOPreviewResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SLOPreviewResponse.ProtoReflect.Descriptor instead. func (*SLOPreviewResponse) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{24} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{25} } func (x *SLOPreviewResponse) GetPlotVector() *PlotVector { @@ -1600,7 +1647,7 @@ type PlotVector struct { func (x *PlotVector) Reset() { *x = PlotVector{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[25] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1613,7 +1660,7 @@ func (x *PlotVector) String() string { func (*PlotVector) ProtoMessage() {} func (x *PlotVector) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[25] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1626,7 +1673,7 @@ func (x *PlotVector) ProtoReflect() protoreflect.Message { // Deprecated: Use PlotVector.ProtoReflect.Descriptor instead. func (*PlotVector) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{25} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{26} } func (x *PlotVector) GetObjective() float64 { @@ -1662,7 +1709,7 @@ type DataPoint struct { func (x *DataPoint) Reset() { *x = DataPoint{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[26] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1675,7 +1722,7 @@ func (x *DataPoint) String() string { func (*DataPoint) ProtoMessage() {} func (x *DataPoint) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[26] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1688,7 +1735,7 @@ func (x *DataPoint) ProtoReflect() protoreflect.Message { // Deprecated: Use DataPoint.ProtoReflect.Descriptor instead. func (*DataPoint) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{26} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{27} } func (x *DataPoint) GetSli() float64 { @@ -1718,7 +1765,7 @@ type AlertFiringWindows struct { func (x *AlertFiringWindows) Reset() { *x = AlertFiringWindows{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[27] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1731,7 +1778,7 @@ func (x *AlertFiringWindows) String() string { func (*AlertFiringWindows) ProtoMessage() {} func (x *AlertFiringWindows) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[27] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1744,7 +1791,7 @@ func (x *AlertFiringWindows) ProtoReflect() protoreflect.Message { // Deprecated: Use AlertFiringWindows.ProtoReflect.Descriptor instead. func (*AlertFiringWindows) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{27} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{28} } func (x *AlertFiringWindows) GetStart() *timestamppb.Timestamp { @@ -1779,7 +1826,7 @@ type ListServiceRequest struct { func (x *ListServiceRequest) Reset() { *x = ListServiceRequest{} if protoimpl.UnsafeEnabled { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[28] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1792,7 +1839,7 @@ func (x *ListServiceRequest) String() string { func (*ListServiceRequest) ProtoMessage() {} func (x *ListServiceRequest) ProtoReflect() protoreflect.Message { - mi := &file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[28] + mi := &file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1805,7 +1852,7 @@ func (x *ListServiceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListServiceRequest.ProtoReflect.Descriptor instead. func (*ListServiceRequest) Descriptor() ([]byte, []int) { - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP(), []int{28} + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP(), []int{29} } func (x *ListServiceRequest) GetDatasource() string { @@ -1815,431 +1862,454 @@ func (x *ListServiceRequest) GetDatasource() string { return "" } -var File_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto protoreflect.FileDescriptor - -var file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDesc = []byte{ - 0x0a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, - 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, - 0x73, 0x2f, 0x73, 0x6c, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x73, 0x6c, 0x6f, 0x2f, 0x73, - 0x6c, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x03, 0x73, 0x6c, 0x6f, 0x1a, 0x1e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, - 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, - 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x68, 0x74, 0x74, 0x70, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, - 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x1a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, +var File_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto protoreflect.FileDescriptor + +var file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDesc = []byte{ + 0x0a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, - 0x69, 0x73, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x17, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x72, 0x70, - 0x63, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x3b, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, - 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, - 0x2f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x76, 0x31, 0x2f, 0x61, 0x6c, 0x65, - 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x45, 0x67, 0x69, 0x74, + 0x69, 0x73, 0x2f, 0x73, 0x6c, 0x6f, 0x2f, 0x76, 0x31, 0x2f, 0x73, 0x6c, 0x6f, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x03, 0x73, 0x6c, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, + 0x70, 0x69, 0x2f, 0x68, 0x74, 0x74, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1c, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, - 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x61, 0x6c, - 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x76, 0x31, 0x2f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, - 0x6e, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x1a, 0x44, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, - 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, - 0x61, 0x70, 0x69, 0x73, 0x2f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x76, 0x31, - 0x2f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, - 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x69, 0x0a, 0x0f, 0x4d, 0x75, 0x6c, 0x74, - 0x69, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x4c, 0x4f, 0x12, 0x29, 0x0a, 0x07, 0x63, - 0x6c, 0x6f, 0x6e, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x07, 0x63, - 0x6c, 0x6f, 0x6e, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x08, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x73, 0x22, 0x32, 0x0a, 0x14, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x43, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x66, - 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x66, - 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x22, 0x8b, 0x01, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, - 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x1c, 0x0a, - 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x49, 0x64, 0x22, 0x70, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x64, - 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x22, 0x53, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, + 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x63, 0x6f, + 0x72, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x1a, 0x17, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, + 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x61, 0x6c, 0x65, 0x72, + 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x76, 0x31, 0x2f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x45, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, + 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, + 0x67, 0x2f, 0x76, 0x31, 0x2f, 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x63, 0x6f, + 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x44, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, + 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, + 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x76, 0x31, 0x2f, 0x61, 0x6c, 0x65, 0x72, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x22, 0x26, 0x0a, 0x0e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x69, 0x0a, 0x0f, 0x4d, + 0x75, 0x6c, 0x74, 0x69, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x4c, 0x4f, 0x12, 0x29, + 0x0a, 0x07, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x52, 0x07, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x08, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x63, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, 0x22, 0x32, 0x0a, 0x14, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x43, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x12, 0x1a, + 0x0a, 0x08, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x08, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x22, 0x8b, 0x01, 0x0a, 0x11, 0x4c, + 0x69, 0x73, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x12, 0x1c, + 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, + 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x64, 0x22, 0x70, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x22, 0x2d, 0x0a, 0x09, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x05, 0x69, 0x74, 0x65, - 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x2d, 0x0a, 0x05, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x22, 0xab, 0x01, 0x0a, 0x0b, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x6a, 0x6f, - 0x62, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, - 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1e, - 0x0a, 0x0a, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x22, - 0x0a, 0x0c, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x64, 0x47, 0x6f, 0x6f, 0x64, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x64, 0x47, 0x6f, - 0x6f, 0x64, 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x64, 0x54, 0x6f, - 0x74, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x49, 0x64, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x22, 0x45, 0x0a, 0x07, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, - 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x02, + 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x22, 0x53, 0x0a, 0x13, 0x4c, 0x69, + 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x22, - 0x31, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x22, - 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, - 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x05, 0x69, 0x74, 0x65, - 0x6d, 0x73, 0x22, 0x1b, 0x0a, 0x05, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, - 0xe7, 0x02, 0x0a, 0x05, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, - 0x12, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6e, 0x6f, 0x74, 0x69, 0x66, - 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x38, 0x0a, - 0x17, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x17, - 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x73, 0x63, - 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, - 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x24, 0x0a, 0x0d, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x54, 0x79, 0x70, 0x65, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, - 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6f, 0x6e, 0x4e, 0x6f, 0x44, 0x61, 0x74, - 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x6f, 0x6e, 0x4e, 0x6f, 0x44, 0x61, 0x74, - 0x61, 0x12, 0x1a, 0x0a, 0x08, 0x6f, 0x6e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x08, 0x6f, 0x6e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x0a, - 0x08, 0x6f, 0x6e, 0x42, 0x72, 0x65, 0x61, 0x63, 0x68, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x08, 0x6f, 0x6e, 0x42, 0x72, 0x65, 0x61, 0x63, 0x68, 0x12, 0x1e, 0x0a, 0x0a, 0x6f, 0x6e, 0x52, - 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x22, 0xae, 0x04, 0x0a, 0x15, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x69, 0x76, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, - 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x49, 0x64, 0x12, 0x26, 0x0a, 0x0e, 0x67, 0x6f, 0x6f, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x67, 0x6f, 0x6f, - 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x74, - 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, - 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x0a, 0x67, 0x6f, 0x6f, 0x64, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x0a, 0x67, 0x6f, 0x6f, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x73, 0x12, 0x2c, 0x0a, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, - 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, - 0x1c, 0x0a, 0x09, 0x73, 0x6c, 0x6f, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x18, 0x09, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x73, 0x6c, 0x6f, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x12, 0x47, 0x0a, - 0x11, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, - 0x61, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x11, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0x69, 0x6e, 0x67, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x23, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x22, 0x0a, 0x06, 0x6c, - 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x73, 0x6c, - 0x6f, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, - 0x49, 0x0a, 0x11, 0x61, 0x74, 0x74, 0x61, 0x63, 0x68, 0x65, 0x64, 0x45, 0x6e, 0x64, 0x70, 0x6f, - 0x69, 0x6e, 0x74, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x61, 0x6c, 0x65, - 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x65, 0x64, 0x45, 0x6e, - 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x52, 0x11, 0x61, 0x74, 0x74, 0x61, 0x63, 0x68, 0x65, - 0x64, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x22, 0x40, 0x0a, 0x10, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x53, 0x4c, 0x4f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2c, - 0x0a, 0x03, 0x73, 0x6c, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6c, - 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x52, 0x03, 0x73, 0x6c, 0x6f, 0x22, 0x35, 0x0a, 0x0d, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x50, 0x61, 0x69, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x24, 0x0a, - 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x73, - 0x6c, 0x6f, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x50, 0x61, 0x69, 0x72, 0x52, 0x05, 0x69, 0x74, - 0x65, 0x6d, 0x73, 0x22, 0x31, 0x0a, 0x09, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x50, 0x61, 0x69, 0x72, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x22, 0x81, 0x01, 0x0a, 0x07, 0x53, 0x4c, 0x4f, 0x44, 0x61, - 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, - 0x69, 0x64, 0x12, 0x2c, 0x0a, 0x03, 0x53, 0x4c, 0x4f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, - 0x65, 0x6c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x52, 0x03, 0x53, 0x4c, 0x4f, - 0x12, 0x38, 0x0a, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, - 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x22, 0x1e, 0x0a, 0x06, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x49, 0x0a, 0x06, 0x4d, 0x65, - 0x74, 0x72, 0x69, 0x63, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x02, 0x69, 0x64, 0x12, 0x2f, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x5a, 0x0a, 0x0e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, - 0x04, 0x75, 0x6e, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x6e, 0x69, - 0x74, 0x22, 0x2f, 0x0a, 0x0a, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4c, 0x69, 0x73, 0x74, 0x12, - 0x21, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, - 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x05, 0x69, 0x74, 0x65, - 0x6d, 0x73, 0x22, 0xc7, 0x01, 0x0a, 0x0f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x47, 0x72, 0x6f, - 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x5c, 0x0a, 0x12, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x4e, - 0x61, 0x6d, 0x65, 0x54, 0x6f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x2e, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, - 0x6d, 0x65, 0x54, 0x6f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x12, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x54, 0x6f, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x1a, 0x56, 0x0a, 0x17, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, 0x6d, - 0x65, 0x54, 0x6f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x25, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x0f, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4c, 0x69, 0x73, - 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x3f, 0x0a, 0x19, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x69, 0x76, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x05, 0x69, 0x74, 0x65, - 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, - 0x4c, 0x4f, 0x44, 0x61, 0x74, 0x61, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x36, 0x0a, - 0x09, 0x53, 0x4c, 0x4f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x29, 0x0a, 0x05, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x13, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, - 0x53, 0x4c, 0x4f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x45, 0x0a, 0x12, 0x53, 0x4c, 0x4f, 0x50, 0x72, 0x65, 0x76, - 0x69, 0x65, 0x77, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x0a, 0x70, - 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0f, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x50, 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x52, 0x0a, 0x70, 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x83, 0x01, 0x0a, - 0x0a, 0x50, 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x6f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, - 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x12, 0x24, 0x0a, 0x05, 0x69, 0x74, 0x65, - 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x44, - 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x12, - 0x31, 0x0a, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x17, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x46, 0x69, 0x72, 0x69, - 0x6e, 0x67, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x52, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, - 0x77, 0x73, 0x22, 0x57, 0x0a, 0x09, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, - 0x10, 0x0a, 0x03, 0x73, 0x6c, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x03, 0x73, 0x6c, - 0x69, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, + 0x2d, 0x0a, 0x09, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x05, + 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x73, 0x6c, + 0x6f, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x2d, + 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x76, 0x61, 0x6c, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x22, 0xab, 0x01, + 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x14, 0x0a, + 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, + 0x62, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, + 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x64, 0x47, 0x6f, 0x6f, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, + 0x64, 0x47, 0x6f, 0x6f, 0x64, 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, + 0x64, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x49, 0x64, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x22, 0x45, 0x0a, 0x07, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, + 0x49, 0x64, 0x22, 0x31, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x69, 0x73, + 0x74, 0x12, 0x22, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x0c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x05, + 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x1b, 0x0a, 0x05, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x22, 0xe7, 0x02, 0x0a, 0x05, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x2e, 0x0a, 0x12, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6e, 0x6f, + 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x12, 0x38, 0x0a, 0x17, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x17, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x44, + 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, + 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x24, 0x0a, 0x0d, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x54, + 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x68, 0x72, 0x65, 0x73, + 0x68, 0x6f, 0x6c, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6f, 0x6e, 0x4e, 0x6f, + 0x44, 0x61, 0x74, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x6f, 0x6e, 0x4e, 0x6f, + 0x44, 0x61, 0x74, 0x61, 0x12, 0x1a, 0x0a, 0x08, 0x6f, 0x6e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x6f, 0x6e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x12, 0x1a, 0x0a, 0x08, 0x6f, 0x6e, 0x42, 0x72, 0x65, 0x61, 0x63, 0x68, 0x18, 0x09, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x08, 0x6f, 0x6e, 0x42, 0x72, 0x65, 0x61, 0x63, 0x68, 0x12, 0x1e, 0x0a, 0x0a, + 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0a, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x22, 0xae, 0x04, 0x0a, + 0x15, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x4f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, + 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x12, 0x26, 0x0a, 0x0e, 0x67, 0x6f, 0x6f, 0x64, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, + 0x67, 0x6f, 0x6f, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x28, + 0x0a, 0x0f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x0a, 0x67, 0x6f, 0x6f, 0x64, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x73, + 0x6c, 0x6f, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x0a, 0x67, 0x6f, 0x6f, 0x64, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2c, 0x0a, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x6c, 0x6f, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x18, + 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x6c, 0x6f, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64, + 0x12, 0x47, 0x0a, 0x11, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x11, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0x69, 0x6e, + 0x67, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x23, 0x0a, 0x06, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x22, + 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, + 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, + 0x6c, 0x73, 0x12, 0x49, 0x0a, 0x11, 0x61, 0x74, 0x74, 0x61, 0x63, 0x68, 0x65, 0x64, 0x45, 0x6e, + 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, + 0x61, 0x6c, 0x65, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x65, + 0x64, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x52, 0x11, 0x61, 0x74, 0x74, 0x61, + 0x63, 0x68, 0x65, 0x64, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x22, 0x40, 0x0a, + 0x10, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x4c, 0x4f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x73, 0x6c, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, + 0x6c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x52, 0x03, 0x73, 0x6c, 0x6f, 0x22, + 0x35, 0x0a, 0x0d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x50, 0x61, 0x69, 0x72, 0x4c, 0x69, 0x73, 0x74, + 0x12, 0x24, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x0e, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x50, 0x61, 0x69, 0x72, 0x52, + 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x31, 0x0a, 0x09, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x50, + 0x61, 0x69, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x04, 0x76, 0x61, 0x6c, 0x73, 0x22, 0x81, 0x01, 0x0a, 0x07, 0x53, 0x4c, + 0x4f, 0x44, 0x61, 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2c, 0x0a, 0x03, 0x53, 0x4c, 0x4f, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x52, 0x03, + 0x53, 0x4c, 0x4f, 0x12, 0x38, 0x0a, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x22, 0x1e, 0x0a, + 0x06, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x49, 0x0a, + 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2f, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x5a, 0x0a, 0x0e, 0x4d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x75, 0x6e, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x75, 0x6e, 0x69, 0x74, 0x22, 0x2f, 0x0a, 0x0a, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x4c, 0x69, + 0x73, 0x74, 0x12, 0x21, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x0b, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x05, + 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0xc7, 0x01, 0x0a, 0x0f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x5c, 0x0a, 0x12, 0x67, 0x72, 0x6f, + 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x54, 0x6f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x2e, 0x47, 0x72, 0x6f, 0x75, + 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x54, 0x6f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x12, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x54, 0x6f, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x1a, 0x56, 0x0a, 0x17, 0x47, 0x72, 0x6f, 0x75, 0x70, + 0x4e, 0x61, 0x6d, 0x65, 0x54, 0x6f, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0x3f, 0x0a, 0x19, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x4f, + 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x05, + 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x73, 0x6c, + 0x6f, 0x2e, 0x53, 0x4c, 0x4f, 0x44, 0x61, 0x74, 0x61, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, + 0x22, 0x36, 0x0a, 0x09, 0x53, 0x4c, 0x4f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x29, 0x0a, + 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x13, 0x2e, 0x73, + 0x6c, 0x6f, 0x2e, 0x53, 0x4c, 0x4f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x45, 0x0a, 0x12, 0x53, 0x4c, 0x4f, 0x50, + 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, + 0x0a, 0x0a, 0x70, 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x50, 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x52, 0x0a, 0x70, 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, + 0x83, 0x01, 0x0a, 0x0a, 0x50, 0x6c, 0x6f, 0x74, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x1c, + 0x0a, 0x09, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x01, 0x52, 0x09, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x12, 0x24, 0x0a, 0x05, + 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x73, 0x6c, + 0x6f, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, + 0x6d, 0x73, 0x12, 0x31, 0x0a, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x46, + 0x69, 0x72, 0x69, 0x6e, 0x67, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x52, 0x07, 0x77, 0x69, + 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x22, 0x57, 0x0a, 0x09, 0x44, 0x61, 0x74, 0x61, 0x50, 0x6f, 0x69, + 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x6c, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, + 0x03, 0x73, 0x6c, 0x69, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x90, + 0x01, 0x0a, 0x12, 0x41, 0x6c, 0x65, 0x72, 0x74, 0x46, 0x69, 0x72, 0x69, 0x6e, 0x67, 0x57, 0x69, + 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x90, 0x01, 0x0a, 0x12, - 0x41, 0x6c, 0x65, 0x72, 0x74, 0x46, 0x69, 0x72, 0x69, 0x6e, 0x67, 0x57, 0x69, 0x6e, 0x64, 0x6f, - 0x77, 0x73, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x65, - 0x6e, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x22, 0x34, - 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x2a, 0x84, 0x01, 0x0a, 0x0e, 0x53, 0x4c, 0x4f, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x49, 0x6e, 0x50, 0x72, 0x6f, - 0x67, 0x72, 0x65, 0x73, 0x73, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x69, 0x6e, 0x67, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x4e, 0x6f, 0x44, 0x61, 0x74, 0x61, 0x10, - 0x02, 0x12, 0x06, 0x0a, 0x02, 0x4f, 0x6b, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x57, 0x61, 0x72, - 0x6e, 0x69, 0x6e, 0x67, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x42, 0x72, 0x65, 0x61, 0x63, 0x68, - 0x69, 0x6e, 0x67, 0x10, 0x05, 0x12, 0x11, 0x0a, 0x0d, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, - 0x44, 0x61, 0x74, 0x61, 0x4f, 0x6b, 0x10, 0x06, 0x12, 0x11, 0x0a, 0x0d, 0x49, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x07, 0x32, 0xb6, 0x07, 0x0a, 0x03, - 0x53, 0x4c, 0x4f, 0x12, 0x45, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x4c, 0x4f, - 0x12, 0x15, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x4c, 0x4f, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, - 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x22, 0x10, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0a, - 0x3a, 0x01, 0x2a, 0x22, 0x05, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x12, 0x3b, 0x0a, 0x06, 0x47, 0x65, - 0x74, 0x53, 0x4c, 0x4f, 0x12, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, + 0x52, 0x03, 0x65, 0x6e, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, + 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, + 0x79, 0x22, 0x34, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, + 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2a, 0x84, 0x01, 0x0a, 0x0e, 0x53, 0x4c, 0x4f, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x49, 0x6e, + 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x4e, 0x6f, 0x44, 0x61, + 0x74, 0x61, 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, 0x4f, 0x6b, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, + 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x42, 0x72, 0x65, + 0x61, 0x63, 0x68, 0x69, 0x6e, 0x67, 0x10, 0x05, 0x12, 0x11, 0x0a, 0x0d, 0x50, 0x61, 0x72, 0x74, + 0x69, 0x61, 0x6c, 0x44, 0x61, 0x74, 0x61, 0x4f, 0x6b, 0x10, 0x06, 0x12, 0x11, 0x0a, 0x0d, 0x49, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x07, 0x32, 0x92, + 0x08, 0x0a, 0x03, 0x53, 0x4c, 0x4f, 0x12, 0x45, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x53, 0x4c, 0x4f, 0x12, 0x15, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x53, 0x4c, 0x4f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0f, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x22, 0x10, 0x82, 0xd3, 0xe4, + 0x93, 0x02, 0x0a, 0x3a, 0x01, 0x2a, 0x22, 0x05, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x12, 0x3b, 0x0a, + 0x06, 0x47, 0x65, 0x74, 0x53, 0x4c, 0x4f, 0x12, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x1a, 0x0c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, + 0x4c, 0x4f, 0x44, 0x61, 0x74, 0x61, 0x22, 0x12, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0c, 0x22, 0x0a, + 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x12, 0x51, 0x0a, 0x08, 0x4c, 0x69, + 0x73, 0x74, 0x53, 0x4c, 0x4f, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1e, + 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, + 0x6c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x0d, + 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x07, 0x12, 0x05, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x12, 0x4f, 0x0a, + 0x09, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x4c, 0x4f, 0x12, 0x0c, 0x2e, 0x73, 0x6c, 0x6f, + 0x2e, 0x53, 0x4c, 0x4f, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x22, 0x1c, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x16, 0x3a, 0x01, 0x2a, 0x1a, 0x11, 0x2f, 0x73, 0x6c, + 0x6f, 0x73, 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x2f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x48, + 0x0a, 0x09, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x4c, 0x4f, 0x12, 0x0f, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x22, 0x12, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0c, 0x2a, 0x0a, 0x2f, 0x73, + 0x6c, 0x6f, 0x73, 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x12, 0x43, 0x0a, 0x08, 0x43, 0x6c, 0x6f, 0x6e, + 0x65, 0x53, 0x4c, 0x4f, 0x12, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x1a, 0x0c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x4c, 0x4f, 0x44, - 0x61, 0x74, 0x61, 0x22, 0x12, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0c, 0x22, 0x0a, 0x2f, 0x73, 0x6c, - 0x6f, 0x73, 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x12, 0x51, 0x0a, 0x08, 0x4c, 0x69, 0x73, 0x74, 0x53, - 0x4c, 0x4f, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1e, 0x2e, 0x73, 0x6c, - 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x0d, 0x82, 0xd3, 0xe4, - 0x93, 0x02, 0x07, 0x12, 0x05, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x12, 0x4f, 0x0a, 0x09, 0x55, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x53, 0x4c, 0x4f, 0x12, 0x0c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x4c, - 0x4f, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x1c, 0x82, - 0xd3, 0xe4, 0x93, 0x02, 0x16, 0x3a, 0x01, 0x2a, 0x1a, 0x11, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x2f, - 0x7b, 0x69, 0x64, 0x7d, 0x2f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x48, 0x0a, 0x09, 0x44, - 0x65, 0x6c, 0x65, 0x74, 0x65, 0x53, 0x4c, 0x4f, 0x12, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x22, 0x12, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0c, 0x2a, 0x0a, 0x2f, 0x73, 0x6c, 0x6f, 0x73, - 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x12, 0x43, 0x0a, 0x08, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x53, 0x4c, - 0x4f, 0x12, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, - 0x63, 0x65, 0x1a, 0x0c, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x4c, 0x4f, 0x44, 0x61, 0x74, 0x61, - 0x22, 0x18, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x12, 0x22, 0x10, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x2f, - 0x7b, 0x69, 0x64, 0x7d, 0x2f, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x12, 0x71, 0x0a, 0x0f, 0x43, 0x6c, - 0x6f, 0x6e, 0x65, 0x54, 0x6f, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, 0x12, 0x14, 0x2e, - 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x53, 0x4c, 0x4f, 0x1a, 0x19, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x43, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x22, 0x2d, - 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x27, 0x3a, 0x01, 0x2a, 0x22, 0x22, 0x2f, 0x73, 0x6c, 0x6f, 0x73, - 0x2f, 0x7b, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x49, 0x64, 0x2e, 0x69, 0x64, 0x7d, 0x2f, 0x63, 0x6c, - 0x6f, 0x6e, 0x65, 0x54, 0x6f, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, 0x12, 0x51, 0x0a, - 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x17, 0x2e, 0x73, - 0x6c, 0x6f, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x65, 0x74, 0x72, - 0x69, 0x63, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x13, 0x82, 0xd3, 0xe4, - 0x93, 0x02, 0x0d, 0x3a, 0x01, 0x2a, 0x22, 0x08, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, - 0x12, 0x50, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x12, 0x18, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x73, 0x6c, 0x6f, - 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x14, 0x82, 0xd3, - 0xe4, 0x93, 0x02, 0x0e, 0x3a, 0x01, 0x2a, 0x22, 0x09, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x73, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, - 0x12, 0x16, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x0f, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x09, - 0x22, 0x07, 0x2f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x44, 0x0a, 0x06, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x0f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x1a, 0x0e, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x4c, 0x4f, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x22, 0x19, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x13, 0x22, 0x11, 0x2f, 0x73, - 0x6c, 0x6f, 0x73, 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x53, 0x0a, 0x07, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x12, 0x15, 0x2e, 0x73, 0x6c, 0x6f, - 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x4c, 0x4f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x17, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x4c, 0x4f, 0x50, 0x72, 0x65, 0x76, 0x69, - 0x65, 0x77, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x18, 0x82, 0xd3, 0xe4, 0x93, - 0x02, 0x12, 0x3a, 0x01, 0x2a, 0x22, 0x0d, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x2f, 0x70, 0x72, 0x65, - 0x76, 0x69, 0x65, 0x77, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, - 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x73, 0x2f, 0x73, 0x6c, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0x73, - 0x2f, 0x73, 0x6c, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x74, 0x61, 0x22, 0x18, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x12, 0x22, 0x10, 0x2f, 0x73, 0x6c, + 0x6f, 0x73, 0x2f, 0x7b, 0x69, 0x64, 0x7d, 0x2f, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x12, 0x71, 0x0a, + 0x0f, 0x43, 0x6c, 0x6f, 0x6e, 0x65, 0x54, 0x6f, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, + 0x12, 0x14, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x43, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x53, 0x4c, 0x4f, 0x1a, 0x19, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, 0x75, 0x6c, + 0x74, 0x69, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, + 0x73, 0x22, 0x2d, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x27, 0x3a, 0x01, 0x2a, 0x22, 0x22, 0x2f, 0x73, + 0x6c, 0x6f, 0x73, 0x2f, 0x7b, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x49, 0x64, 0x2e, 0x69, 0x64, 0x7d, + 0x2f, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x54, 0x6f, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, + 0x12, 0x51, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, + 0x17, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x13, + 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0d, 0x3a, 0x01, 0x2a, 0x22, 0x08, 0x2f, 0x6d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x73, 0x12, 0x50, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x12, 0x18, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, + 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x22, + 0x14, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0e, 0x3a, 0x01, 0x2a, 0x22, 0x09, 0x2f, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x73, 0x12, 0x16, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x73, 0x6c, + 0x6f, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x0f, 0x82, 0xd3, 0xe4, + 0x93, 0x02, 0x09, 0x22, 0x07, 0x2f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x4a, 0x0a, 0x0c, + 0x47, 0x65, 0x74, 0x53, 0x4c, 0x4f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0f, 0x2e, 0x63, + 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x1a, 0x0e, 0x2e, + 0x73, 0x6c, 0x6f, 0x2e, 0x53, 0x4c, 0x4f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x19, 0x82, + 0xd3, 0xe4, 0x93, 0x02, 0x13, 0x22, 0x11, 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x2f, 0x7b, 0x69, 0x64, + 0x7d, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x53, 0x0a, 0x07, 0x50, 0x72, 0x65, 0x76, + 0x69, 0x65, 0x77, 0x12, 0x15, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x53, 0x4c, 0x4f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x73, 0x6c, 0x6f, + 0x2e, 0x53, 0x4c, 0x4f, 0x50, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x18, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x12, 0x3a, 0x01, 0x2a, 0x22, 0x0d, + 0x2f, 0x73, 0x6c, 0x6f, 0x73, 0x2f, 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x12, 0x54, 0x0a, + 0x0f, 0x4c, 0x69, 0x73, 0x74, 0x44, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, + 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x13, 0x2e, 0x73, 0x6c, 0x6f, 0x2e, 0x44, + 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x14, 0x82, + 0xd3, 0xe4, 0x93, 0x02, 0x0e, 0x12, 0x0c, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x73, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x65, 0x72, 0x2f, 0x6f, 0x70, 0x6e, 0x69, 0x2f, 0x70, + 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x73, 0x6c, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescOnce sync.Once - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescData = file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDesc + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescOnce sync.Once + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescData = file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDesc ) -func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescGZIP() []byte { - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescOnce.Do(func() { - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescData = protoimpl.X.CompressGZIP(file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescData) +func file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescGZIP() []byte { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescOnce.Do(func() { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescData = protoimpl.X.CompressGZIP(file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescData) }) - return file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDescData + return file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDescData } -var file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes = make([]protoimpl.MessageInfo, 30) -var file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_goTypes = []interface{}{ +var file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes = make([]protoimpl.MessageInfo, 31) +var file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_goTypes = []interface{}{ (SLOStatusState)(0), // 0: slo.SLOStatusState - (*MultiClusterSLO)(nil), // 1: slo.MultiClusterSLO - (*MultiClusterFailures)(nil), // 2: slo.MultiClusterFailures - (*ListEventsRequest)(nil), // 3: slo.ListEventsRequest - (*ListMetricsRequest)(nil), // 4: slo.ListMetricsRequest - (*ListServicesRequest)(nil), // 5: slo.ListServicesRequest - (*EventList)(nil), // 6: slo.EventList - (*Event)(nil), // 7: slo.Event - (*ServiceInfo)(nil), // 8: slo.ServiceInfo - (*Service)(nil), // 9: slo.Service - (*ServiceList)(nil), // 10: slo.ServiceList - (*Label)(nil), // 11: slo.Label - (*Alert)(nil), // 12: slo.Alert - (*ServiceLevelObjective)(nil), // 13: slo.ServiceLevelObjective - (*CreateSLORequest)(nil), // 14: slo.CreateSLORequest - (*EventPairList)(nil), // 15: slo.EventPairList - (*EventPair)(nil), // 16: slo.EventPair - (*SLOData)(nil), // 17: slo.SLOData - (*Target)(nil), // 18: slo.Target - (*Metric)(nil), // 19: slo.Metric - (*MetricMetadata)(nil), // 20: slo.MetricMetadata - (*MetricList)(nil), // 21: slo.MetricList - (*MetricGroupList)(nil), // 22: slo.MetricGroupList - (*ServiceLevelObjectiveList)(nil), // 23: slo.ServiceLevelObjectiveList - (*SLOStatus)(nil), // 24: slo.SLOStatus - (*SLOPreviewResponse)(nil), // 25: slo.SLOPreviewResponse - (*PlotVector)(nil), // 26: slo.PlotVector - (*DataPoint)(nil), // 27: slo.DataPoint - (*AlertFiringWindows)(nil), // 28: slo.AlertFiringWindows - (*ListServiceRequest)(nil), // 29: slo.ListServiceRequest - nil, // 30: slo.MetricGroupList.GroupNameToMetricsEntry - (*v1.Reference)(nil), // 31: core.Reference - (*durationpb.Duration)(nil), // 32: google.protobuf.Duration - (*v11.AttachedEndpoints)(nil), // 33: alerting.AttachedEndpoints - (*timestamppb.Timestamp)(nil), // 34: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 35: google.protobuf.Empty -} -var file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_depIdxs = []int32{ - 31, // 0: slo.MultiClusterSLO.cloneId:type_name -> core.Reference - 31, // 1: slo.MultiClusterSLO.clusters:type_name -> core.Reference - 7, // 2: slo.EventList.items:type_name -> slo.Event - 9, // 3: slo.ServiceList.items:type_name -> slo.Service - 7, // 4: slo.ServiceLevelObjective.goodEvents:type_name -> slo.Event - 7, // 5: slo.ServiceLevelObjective.totalEvents:type_name -> slo.Event - 32, // 6: slo.ServiceLevelObjective.budgetingInterval:type_name -> google.protobuf.Duration - 18, // 7: slo.ServiceLevelObjective.target:type_name -> slo.Target - 11, // 8: slo.ServiceLevelObjective.labels:type_name -> slo.Label - 33, // 9: slo.ServiceLevelObjective.attachedEndpoints:type_name -> alerting.AttachedEndpoints - 13, // 10: slo.CreateSLORequest.slo:type_name -> slo.ServiceLevelObjective - 16, // 11: slo.EventPairList.items:type_name -> slo.EventPair - 13, // 12: slo.SLOData.SLO:type_name -> slo.ServiceLevelObjective - 34, // 13: slo.SLOData.createdAt:type_name -> google.protobuf.Timestamp - 20, // 14: slo.Metric.metadata:type_name -> slo.MetricMetadata - 19, // 15: slo.MetricList.items:type_name -> slo.Metric - 30, // 16: slo.MetricGroupList.groupNameToMetrics:type_name -> slo.MetricGroupList.GroupNameToMetricsEntry - 17, // 17: slo.ServiceLevelObjectiveList.items:type_name -> slo.SLOData + (*DatasourceInfo)(nil), // 1: slo.DatasourceInfo + (*MultiClusterSLO)(nil), // 2: slo.MultiClusterSLO + (*MultiClusterFailures)(nil), // 3: slo.MultiClusterFailures + (*ListEventsRequest)(nil), // 4: slo.ListEventsRequest + (*ListMetricsRequest)(nil), // 5: slo.ListMetricsRequest + (*ListServicesRequest)(nil), // 6: slo.ListServicesRequest + (*EventList)(nil), // 7: slo.EventList + (*Event)(nil), // 8: slo.Event + (*ServiceInfo)(nil), // 9: slo.ServiceInfo + (*Service)(nil), // 10: slo.Service + (*ServiceList)(nil), // 11: slo.ServiceList + (*Label)(nil), // 12: slo.Label + (*Alert)(nil), // 13: slo.Alert + (*ServiceLevelObjective)(nil), // 14: slo.ServiceLevelObjective + (*CreateSLORequest)(nil), // 15: slo.CreateSLORequest + (*EventPairList)(nil), // 16: slo.EventPairList + (*EventPair)(nil), // 17: slo.EventPair + (*SLOData)(nil), // 18: slo.SLOData + (*Target)(nil), // 19: slo.Target + (*Metric)(nil), // 20: slo.Metric + (*MetricMetadata)(nil), // 21: slo.MetricMetadata + (*MetricList)(nil), // 22: slo.MetricList + (*MetricGroupList)(nil), // 23: slo.MetricGroupList + (*ServiceLevelObjectiveList)(nil), // 24: slo.ServiceLevelObjectiveList + (*SLOStatus)(nil), // 25: slo.SLOStatus + (*SLOPreviewResponse)(nil), // 26: slo.SLOPreviewResponse + (*PlotVector)(nil), // 27: slo.PlotVector + (*DataPoint)(nil), // 28: slo.DataPoint + (*AlertFiringWindows)(nil), // 29: slo.AlertFiringWindows + (*ListServiceRequest)(nil), // 30: slo.ListServiceRequest + nil, // 31: slo.MetricGroupList.GroupNameToMetricsEntry + (*v1.Reference)(nil), // 32: core.Reference + (*durationpb.Duration)(nil), // 33: google.protobuf.Duration + (*v11.AttachedEndpoints)(nil), // 34: alerting.AttachedEndpoints + (*timestamppb.Timestamp)(nil), // 35: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 36: google.protobuf.Empty +} +var file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_depIdxs = []int32{ + 32, // 0: slo.MultiClusterSLO.cloneId:type_name -> core.Reference + 32, // 1: slo.MultiClusterSLO.clusters:type_name -> core.Reference + 8, // 2: slo.EventList.items:type_name -> slo.Event + 10, // 3: slo.ServiceList.items:type_name -> slo.Service + 8, // 4: slo.ServiceLevelObjective.goodEvents:type_name -> slo.Event + 8, // 5: slo.ServiceLevelObjective.totalEvents:type_name -> slo.Event + 33, // 6: slo.ServiceLevelObjective.budgetingInterval:type_name -> google.protobuf.Duration + 19, // 7: slo.ServiceLevelObjective.target:type_name -> slo.Target + 12, // 8: slo.ServiceLevelObjective.labels:type_name -> slo.Label + 34, // 9: slo.ServiceLevelObjective.attachedEndpoints:type_name -> alerting.AttachedEndpoints + 14, // 10: slo.CreateSLORequest.slo:type_name -> slo.ServiceLevelObjective + 17, // 11: slo.EventPairList.items:type_name -> slo.EventPair + 14, // 12: slo.SLOData.SLO:type_name -> slo.ServiceLevelObjective + 35, // 13: slo.SLOData.createdAt:type_name -> google.protobuf.Timestamp + 21, // 14: slo.Metric.metadata:type_name -> slo.MetricMetadata + 20, // 15: slo.MetricList.items:type_name -> slo.Metric + 31, // 16: slo.MetricGroupList.groupNameToMetrics:type_name -> slo.MetricGroupList.GroupNameToMetricsEntry + 18, // 17: slo.ServiceLevelObjectiveList.items:type_name -> slo.SLOData 0, // 18: slo.SLOStatus.state:type_name -> slo.SLOStatusState - 26, // 19: slo.SLOPreviewResponse.plotVector:type_name -> slo.PlotVector - 27, // 20: slo.PlotVector.items:type_name -> slo.DataPoint - 28, // 21: slo.PlotVector.windows:type_name -> slo.AlertFiringWindows - 34, // 22: slo.DataPoint.timestamp:type_name -> google.protobuf.Timestamp - 34, // 23: slo.AlertFiringWindows.start:type_name -> google.protobuf.Timestamp - 34, // 24: slo.AlertFiringWindows.end:type_name -> google.protobuf.Timestamp - 21, // 25: slo.MetricGroupList.GroupNameToMetricsEntry.value:type_name -> slo.MetricList - 14, // 26: slo.SLO.CreateSLO:input_type -> slo.CreateSLORequest - 31, // 27: slo.SLO.GetSLO:input_type -> core.Reference - 35, // 28: slo.SLO.ListSLOs:input_type -> google.protobuf.Empty - 17, // 29: slo.SLO.UpdateSLO:input_type -> slo.SLOData - 31, // 30: slo.SLO.DeleteSLO:input_type -> core.Reference - 31, // 31: slo.SLO.CloneSLO:input_type -> core.Reference - 1, // 32: slo.SLO.CloneToClusters:input_type -> slo.MultiClusterSLO - 4, // 33: slo.SLO.ListMetrics:input_type -> slo.ListMetricsRequest - 5, // 34: slo.SLO.ListServices:input_type -> slo.ListServicesRequest - 3, // 35: slo.SLO.ListEvents:input_type -> slo.ListEventsRequest - 31, // 36: slo.SLO.Status:input_type -> core.Reference - 14, // 37: slo.SLO.Preview:input_type -> slo.CreateSLORequest - 31, // 38: slo.SLO.CreateSLO:output_type -> core.Reference - 17, // 39: slo.SLO.GetSLO:output_type -> slo.SLOData - 23, // 40: slo.SLO.ListSLOs:output_type -> slo.ServiceLevelObjectiveList - 35, // 41: slo.SLO.UpdateSLO:output_type -> google.protobuf.Empty - 35, // 42: slo.SLO.DeleteSLO:output_type -> google.protobuf.Empty - 17, // 43: slo.SLO.CloneSLO:output_type -> slo.SLOData - 2, // 44: slo.SLO.CloneToClusters:output_type -> slo.MultiClusterFailures - 22, // 45: slo.SLO.ListMetrics:output_type -> slo.MetricGroupList - 10, // 46: slo.SLO.ListServices:output_type -> slo.ServiceList - 6, // 47: slo.SLO.ListEvents:output_type -> slo.EventList - 24, // 48: slo.SLO.Status:output_type -> slo.SLOStatus - 25, // 49: slo.SLO.Preview:output_type -> slo.SLOPreviewResponse - 38, // [38:50] is the sub-list for method output_type - 26, // [26:38] is the sub-list for method input_type + 27, // 19: slo.SLOPreviewResponse.plotVector:type_name -> slo.PlotVector + 28, // 20: slo.PlotVector.items:type_name -> slo.DataPoint + 29, // 21: slo.PlotVector.windows:type_name -> slo.AlertFiringWindows + 35, // 22: slo.DataPoint.timestamp:type_name -> google.protobuf.Timestamp + 35, // 23: slo.AlertFiringWindows.start:type_name -> google.protobuf.Timestamp + 35, // 24: slo.AlertFiringWindows.end:type_name -> google.protobuf.Timestamp + 22, // 25: slo.MetricGroupList.GroupNameToMetricsEntry.value:type_name -> slo.MetricList + 15, // 26: slo.SLO.CreateSLO:input_type -> slo.CreateSLORequest + 32, // 27: slo.SLO.GetSLO:input_type -> core.Reference + 36, // 28: slo.SLO.ListSLOs:input_type -> google.protobuf.Empty + 18, // 29: slo.SLO.UpdateSLO:input_type -> slo.SLOData + 32, // 30: slo.SLO.DeleteSLO:input_type -> core.Reference + 32, // 31: slo.SLO.CloneSLO:input_type -> core.Reference + 2, // 32: slo.SLO.CloneToClusters:input_type -> slo.MultiClusterSLO + 5, // 33: slo.SLO.ListMetrics:input_type -> slo.ListMetricsRequest + 6, // 34: slo.SLO.ListServices:input_type -> slo.ListServicesRequest + 4, // 35: slo.SLO.ListEvents:input_type -> slo.ListEventsRequest + 32, // 36: slo.SLO.GetSLOStatus:input_type -> core.Reference + 15, // 37: slo.SLO.Preview:input_type -> slo.CreateSLORequest + 36, // 38: slo.SLO.ListDatasources:input_type -> google.protobuf.Empty + 32, // 39: slo.SLO.CreateSLO:output_type -> core.Reference + 18, // 40: slo.SLO.GetSLO:output_type -> slo.SLOData + 24, // 41: slo.SLO.ListSLOs:output_type -> slo.ServiceLevelObjectiveList + 36, // 42: slo.SLO.UpdateSLO:output_type -> google.protobuf.Empty + 36, // 43: slo.SLO.DeleteSLO:output_type -> google.protobuf.Empty + 18, // 44: slo.SLO.CloneSLO:output_type -> slo.SLOData + 3, // 45: slo.SLO.CloneToClusters:output_type -> slo.MultiClusterFailures + 23, // 46: slo.SLO.ListMetrics:output_type -> slo.MetricGroupList + 11, // 47: slo.SLO.ListServices:output_type -> slo.ServiceList + 7, // 48: slo.SLO.ListEvents:output_type -> slo.EventList + 25, // 49: slo.SLO.GetSLOStatus:output_type -> slo.SLOStatus + 26, // 50: slo.SLO.Preview:output_type -> slo.SLOPreviewResponse + 1, // 51: slo.SLO.ListDatasources:output_type -> slo.DatasourceInfo + 39, // [39:52] is the sub-list for method output_type + 26, // [26:39] is the sub-list for method input_type 26, // [26:26] is the sub-list for extension type_name 26, // [26:26] is the sub-list for extension extendee 0, // [0:26] is the sub-list for field type_name } -func init() { file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() } -func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { - if File_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto != nil { +func init() { file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_init() } +func file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_init() { + if File_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto != nil { return } if !protoimpl.UnsafeEnabled { - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DatasourceInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*MultiClusterSLO); i { case 0: return &v.state @@ -2251,7 +2321,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*MultiClusterFailures); i { case 0: return &v.state @@ -2263,7 +2333,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListEventsRequest); i { case 0: return &v.state @@ -2275,7 +2345,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListMetricsRequest); i { case 0: return &v.state @@ -2287,7 +2357,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListServicesRequest); i { case 0: return &v.state @@ -2299,7 +2369,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*EventList); i { case 0: return &v.state @@ -2311,7 +2381,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Event); i { case 0: return &v.state @@ -2323,7 +2393,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ServiceInfo); i { case 0: return &v.state @@ -2335,7 +2405,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Service); i { case 0: return &v.state @@ -2347,7 +2417,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ServiceList); i { case 0: return &v.state @@ -2359,7 +2429,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Label); i { case 0: return &v.state @@ -2371,7 +2441,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Alert); i { case 0: return &v.state @@ -2383,7 +2453,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ServiceLevelObjective); i { case 0: return &v.state @@ -2395,7 +2465,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*CreateSLORequest); i { case 0: return &v.state @@ -2407,7 +2477,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*EventPairList); i { case 0: return &v.state @@ -2419,7 +2489,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*EventPair); i { case 0: return &v.state @@ -2431,7 +2501,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SLOData); i { case 0: return &v.state @@ -2443,7 +2513,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Target); i { case 0: return &v.state @@ -2455,7 +2525,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Metric); i { case 0: return &v.state @@ -2467,7 +2537,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*MetricMetadata); i { case 0: return &v.state @@ -2479,7 +2549,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*MetricList); i { case 0: return &v.state @@ -2491,7 +2561,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*MetricGroupList); i { case 0: return &v.state @@ -2503,7 +2573,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ServiceLevelObjectiveList); i { case 0: return &v.state @@ -2515,7 +2585,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SLOStatus); i { case 0: return &v.state @@ -2527,7 +2597,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SLOPreviewResponse); i { case 0: return &v.state @@ -2539,7 +2609,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*PlotVector); i { case 0: return &v.state @@ -2551,7 +2621,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DataPoint); i { case 0: return &v.state @@ -2563,7 +2633,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*AlertFiringWindows); i { case 0: return &v.state @@ -2575,7 +2645,7 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { return nil } } - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListServiceRequest); i { case 0: return &v.state @@ -2592,19 +2662,19 @@ func file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_init() { out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDesc, + RawDescriptor: file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDesc, NumEnums: 1, - NumMessages: 30, + NumMessages: 31, NumExtensions: 0, NumServices: 1, }, - GoTypes: file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_goTypes, - DependencyIndexes: file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_depIdxs, - EnumInfos: file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_enumTypes, - MessageInfos: file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_msgTypes, + GoTypes: file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_goTypes, + DependencyIndexes: file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_depIdxs, + EnumInfos: file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_enumTypes, + MessageInfos: file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_msgTypes, }.Build() - File_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto = out.File - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_rawDesc = nil - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_goTypes = nil - file_github_com_rancher_opni_plugins_slo_apis_slo_slo_proto_depIdxs = nil + File_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto = out.File + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_rawDesc = nil + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_goTypes = nil + file_github_com_rancher_opni_pkg_apis_slo_v1_slo_proto_depIdxs = nil } diff --git a/plugins/slo/apis/slo/slo.proto b/pkg/apis/slo/v1/slo.proto similarity index 93% rename from plugins/slo/apis/slo/slo.proto rename to pkg/apis/slo/v1/slo.proto index dd6a83934b..3b8abfa6c6 100644 --- a/plugins/slo/apis/slo/slo.proto +++ b/pkg/apis/slo/v1/slo.proto @@ -1,5 +1,5 @@ syntax = "proto3"; -option go_package = "github.com/rancher/opni/plugins/slo/apis/slo"; +option go_package = "github.com/rancher/opni/pkg/apis/slo/v1"; import "google/protobuf/duration.proto"; import "google/protobuf/empty.proto"; @@ -80,7 +80,7 @@ service SLO { } // Returns a status enum badge for a given SLO - rpc Status(core.Reference) returns (SLOStatus) { + rpc GetSLOStatus(core.Reference) returns (SLOStatus) { option (google.api.http) = { post: "/slos/{id}/status" @@ -93,8 +93,19 @@ service SLO { body: "*" }; } + + rpc ListDatasources(google.protobuf.Empty) returns (DatasourceInfo) { + option (google.api.http) = { + get: "/datasources" + }; + } } +message DatasourceInfo { + repeated string items = 1; +} + + message MultiClusterSLO { core.Reference cloneId = 1; repeated core.Reference clusters = 2; @@ -256,6 +267,7 @@ message PlotVector { } message DataPoint { + // expected to be between 0 and 100 double sli = 1; google.protobuf.Timestamp timestamp = 2; } diff --git a/plugins/slo/apis/slo/slo_grpc.pb.go b/pkg/apis/slo/v1/slo_grpc.pb.go similarity index 87% rename from plugins/slo/apis/slo/slo_grpc.pb.go rename to pkg/apis/slo/v1/slo_grpc.pb.go index 51247d3124..cbdf023d72 100644 --- a/plugins/slo/apis/slo/slo_grpc.pb.go +++ b/pkg/apis/slo/v1/slo_grpc.pb.go @@ -2,9 +2,9 @@ // versions: // - protoc-gen-go-grpc v1.3.0 // - ragu v1.0.0 -// source: github.com/rancher/opni/plugins/slo/apis/slo/slo.proto +// source: github.com/rancher/opni/pkg/apis/slo/v1/slo.proto -package slo +package v1 import ( context "context" @@ -31,8 +31,9 @@ const ( SLO_ListMetrics_FullMethodName = "/slo.SLO/ListMetrics" SLO_ListServices_FullMethodName = "/slo.SLO/ListServices" SLO_ListEvents_FullMethodName = "/slo.SLO/ListEvents" - SLO_Status_FullMethodName = "/slo.SLO/Status" + SLO_GetSLOStatus_FullMethodName = "/slo.SLO/GetSLOStatus" SLO_Preview_FullMethodName = "/slo.SLO/Preview" + SLO_ListDatasources_FullMethodName = "/slo.SLO/ListDatasources" ) // SLOClient is the client API for SLO service. @@ -53,8 +54,9 @@ type SLOClient interface { ListServices(ctx context.Context, in *ListServicesRequest, opts ...grpc.CallOption) (*ServiceList, error) ListEvents(ctx context.Context, in *ListEventsRequest, opts ...grpc.CallOption) (*EventList, error) // Returns a status enum badge for a given SLO - Status(ctx context.Context, in *v1.Reference, opts ...grpc.CallOption) (*SLOStatus, error) + GetSLOStatus(ctx context.Context, in *v1.Reference, opts ...grpc.CallOption) (*SLOStatus, error) Preview(ctx context.Context, in *CreateSLORequest, opts ...grpc.CallOption) (*SLOPreviewResponse, error) + ListDatasources(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*DatasourceInfo, error) } type sLOClient struct { @@ -155,9 +157,9 @@ func (c *sLOClient) ListEvents(ctx context.Context, in *ListEventsRequest, opts return out, nil } -func (c *sLOClient) Status(ctx context.Context, in *v1.Reference, opts ...grpc.CallOption) (*SLOStatus, error) { +func (c *sLOClient) GetSLOStatus(ctx context.Context, in *v1.Reference, opts ...grpc.CallOption) (*SLOStatus, error) { out := new(SLOStatus) - err := c.cc.Invoke(ctx, SLO_Status_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, SLO_GetSLOStatus_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -173,6 +175,15 @@ func (c *sLOClient) Preview(ctx context.Context, in *CreateSLORequest, opts ...g return out, nil } +func (c *sLOClient) ListDatasources(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*DatasourceInfo, error) { + out := new(DatasourceInfo) + err := c.cc.Invoke(ctx, SLO_ListDatasources_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // SLOServer is the server API for SLO service. // All implementations must embed UnimplementedSLOServer // for forward compatibility @@ -191,8 +202,9 @@ type SLOServer interface { ListServices(context.Context, *ListServicesRequest) (*ServiceList, error) ListEvents(context.Context, *ListEventsRequest) (*EventList, error) // Returns a status enum badge for a given SLO - Status(context.Context, *v1.Reference) (*SLOStatus, error) + GetSLOStatus(context.Context, *v1.Reference) (*SLOStatus, error) Preview(context.Context, *CreateSLORequest) (*SLOPreviewResponse, error) + ListDatasources(context.Context, *emptypb.Empty) (*DatasourceInfo, error) mustEmbedUnimplementedSLOServer() } @@ -230,12 +242,15 @@ func (UnimplementedSLOServer) ListServices(context.Context, *ListServicesRequest func (UnimplementedSLOServer) ListEvents(context.Context, *ListEventsRequest) (*EventList, error) { return nil, status.Errorf(codes.Unimplemented, "method ListEvents not implemented") } -func (UnimplementedSLOServer) Status(context.Context, *v1.Reference) (*SLOStatus, error) { - return nil, status.Errorf(codes.Unimplemented, "method Status not implemented") +func (UnimplementedSLOServer) GetSLOStatus(context.Context, *v1.Reference) (*SLOStatus, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetSLOStatus not implemented") } func (UnimplementedSLOServer) Preview(context.Context, *CreateSLORequest) (*SLOPreviewResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Preview not implemented") } +func (UnimplementedSLOServer) ListDatasources(context.Context, *emptypb.Empty) (*DatasourceInfo, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListDatasources not implemented") +} func (UnimplementedSLOServer) mustEmbedUnimplementedSLOServer() {} // UnsafeSLOServer may be embedded to opt out of forward compatibility for this service. @@ -429,20 +444,20 @@ func _SLO_ListEvents_Handler(srv interface{}, ctx context.Context, dec func(inte return interceptor(ctx, in, info, handler) } -func _SLO_Status_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _SLO_GetSLOStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(v1.Reference) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(SLOServer).Status(ctx, in) + return srv.(SLOServer).GetSLOStatus(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: SLO_Status_FullMethodName, + FullMethod: SLO_GetSLOStatus_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SLOServer).Status(ctx, req.(*v1.Reference)) + return srv.(SLOServer).GetSLOStatus(ctx, req.(*v1.Reference)) } return interceptor(ctx, in, info, handler) } @@ -465,6 +480,24 @@ func _SLO_Preview_Handler(srv interface{}, ctx context.Context, dec func(interfa return interceptor(ctx, in, info, handler) } +func _SLO_ListDatasources_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(emptypb.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SLOServer).ListDatasources(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: SLO_ListDatasources_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SLOServer).ListDatasources(ctx, req.(*emptypb.Empty)) + } + return interceptor(ctx, in, info, handler) +} + // SLO_ServiceDesc is the grpc.ServiceDesc for SLO service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -513,14 +546,18 @@ var SLO_ServiceDesc = grpc.ServiceDesc{ Handler: _SLO_ListEvents_Handler, }, { - MethodName: "Status", - Handler: _SLO_Status_Handler, + MethodName: "GetSLOStatus", + Handler: _SLO_GetSLOStatus_Handler, }, { MethodName: "Preview", Handler: _SLO_Preview_Handler, }, + { + MethodName: "ListDatasources", + Handler: _SLO_ListDatasources_Handler, + }, }, Streams: []grpc.StreamDesc{}, - Metadata: "github.com/rancher/opni/plugins/slo/apis/slo/slo.proto", + Metadata: "github.com/rancher/opni/pkg/apis/slo/v1/slo.proto", } diff --git a/plugins/slo/apis/slo/validate.go b/pkg/apis/slo/v1/validate.go similarity index 92% rename from plugins/slo/apis/slo/validate.go rename to pkg/apis/slo/v1/validate.go index 68a45bc7e6..38e1e21ab7 100644 --- a/plugins/slo/apis/slo/validate.go +++ b/pkg/apis/slo/v1/validate.go @@ -1,4 +1,4 @@ -package slo +package v1 import ( "time" @@ -26,9 +26,6 @@ func (slo *ServiceLevelObjective) Validate() error { if err != nil { return validation.Error("Passed in sloPeriod string is not a valid prometheus duration") } - if slo.Datasource != shared.MonitoringDatasource && slo.Datasource != shared.LoggingDatasource { - return shared.ErrInvalidDatasource - } if slo.GetName() == "" { return validation.Error("slo name must be set") } @@ -118,9 +115,6 @@ func (l *ListEventsRequest) Validate() error { if l.GetDatasource() == "" { return validation.Error("datasource must be set") } - if l.GetDatasource() != shared.MonitoringDatasource && l.GetDatasource() != shared.LoggingDatasource { - return shared.ErrInvalidDatasource - } if l.GetServiceId() == "" { return validation.Error("serviceId must be set") } diff --git a/pkg/slo/query/query_suite_test.go b/pkg/metrics/compat/compat_suite_test.go similarity index 65% rename from pkg/slo/query/query_suite_test.go rename to pkg/metrics/compat/compat_suite_test.go index 9a6ffd5be7..e6bf576460 100644 --- a/pkg/slo/query/query_suite_test.go +++ b/pkg/metrics/compat/compat_suite_test.go @@ -1,4 +1,4 @@ -package query_test +package compat_test import ( "testing" @@ -8,7 +8,7 @@ import ( _ "github.com/rancher/opni/pkg/test/setup" ) -func TestQuery(t *testing.T) { +func TestCompat(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Query Suite") + RunSpecs(t, "Compat Suite") } diff --git a/pkg/metrics/compat/unmarshal.go b/pkg/metrics/compat/unmarshal.go index fc4f8bf57c..5c75c1c646 100644 --- a/pkg/metrics/compat/unmarshal.go +++ b/pkg/metrics/compat/unmarshal.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "net/http" + "slices" "github.com/prometheus/common/model" "go.uber.org/zap" @@ -20,7 +21,7 @@ type QueryResult struct { type ErrorType string // struct for unmarshalling from prometheus api responses -type apiResponse struct { +type ApiResponse struct { Status string `json:"status"` Data json.RawMessage `json:"data"` ErrorType ErrorType `json:"errorType"` @@ -85,8 +86,60 @@ func (qr *QueryResult) GetMatrix() (*model.Matrix, error) { } } +func (qr *QueryResult) GetScalar() (*model.Scalar, error) { + switch qr.V.Type() { + case model.ValScalar: + v := *qr.V.(*model.Scalar) + return &v, nil + default: + return nil, fmt.Errorf("cannot unmarshal prometheus response into scalar type") + } +} + +type Sample struct { + Value float64 + Timestamp int64 +} + +func (qr *QueryResult) LinearSamples() []Sample { + res := []Sample{} + switch qr.V.Type() { + case model.ValVector: + v := qr.V.(model.Vector) + for _, sample := range v { + res = append(res, Sample{ + Value: float64(sample.Value), + Timestamp: int64(sample.Timestamp), + }) + } + case model.ValScalar: + v := qr.V.(*model.Scalar) + res = append(res, Sample{ + Value: float64(v.Value), + Timestamp: int64(v.Timestamp), + }) + case model.ValMatrix: + sampleStream := qr.V.(model.Matrix) + for _, sample := range sampleStream { + for _, s := range sample.Values { + res = append(res, Sample{ + Value: float64(s.Value), + Timestamp: int64(s.Timestamp), + }) + } + } + default: + panic("bug: unreachable code") + } + + slices.SortFunc(res, func(i, j Sample) int { + return int(i.Timestamp - j.Timestamp) + }) + return res +} + func UnmarshalPrometheusResponse(data []byte) (*QueryResult, error) { - var a apiResponse + var a ApiResponse var q QueryResult if err := json.Unmarshal(data, &a); err != nil { @@ -130,15 +183,6 @@ type Response struct { Warnings []string `json:"warnings,omitempty"` } -func unmarshallPrometheusWebResponseData(data []byte) (*Response, error) { - var r Response - err := json.Unmarshal(data, &r) - if err != nil { - return nil, err - } - return &r, nil -} - func UnmarshallPrometheusWebResponse(resp *http.Response, _ *zap.SugaredLogger) (*Response, error) { var val *Response err := json.NewDecoder(resp.Body).Decode(&val) diff --git a/pkg/metrics/compat/unmarshal_test.go b/pkg/metrics/compat/unmarshal_test.go new file mode 100644 index 0000000000..3eb20bc518 --- /dev/null +++ b/pkg/metrics/compat/unmarshal_test.go @@ -0,0 +1,198 @@ +package compat_test + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/common/model" + "github.com/rancher/opni/pkg/metrics/compat" +) + +var _ = Describe("Prometheus Query response unmarshalling", Label("unit"), func() { + It("should unmarshal scalar responses", func() { + scalarData := model.Scalar{ + Value: 1, + Timestamp: model.Time(1), + } + + scalarDataWrapper := struct { + Type model.ValueType `json:"resultType"` + Result json.RawMessage `json:"result"` + }{} + scalarDataWrapper.Type = model.ValScalar + scalarDataBytes, err := json.Marshal(scalarData) + Expect(err).To(Succeed()) + scalarDataWrapper.Result = scalarDataBytes + + scalarDataWrapperBytes, err := json.Marshal(scalarDataWrapper) + Expect(err).To(Succeed()) + + promResp := compat.ApiResponse{ + Status: "success", + Data: scalarDataWrapperBytes, + ErrorType: "", + Error: "", + Warnings: []string{}, + } + + promRespData, err := json.Marshal(promResp) + Expect(err).To(Succeed()) + + qr, err := compat.UnmarshalPrometheusResponse(promRespData) + Expect(err).To(Succeed()) + Expect(qr.Type).To(Equal(model.ValScalar)) + Expect(qr.V).To(Equal(&scalarData)) + + vector, err := qr.GetVector() + Expect(err).NotTo(Succeed()) + Expect(vector).To(BeNil()) + + matrix, err := qr.GetMatrix() + Expect(err).NotTo(Succeed()) + Expect(matrix).To(BeNil()) + + scalar, err := qr.GetScalar() + Expect(err).To(Succeed()) + Expect(scalar.Value).To(Equal(scalarData.Value)) + Expect(scalar.Timestamp).To(Equal(scalarData.Timestamp)) + + samples := qr.LinearSamples() + Expect(samples).To(HaveLen(1)) + Expect(samples[0].Value).To(Equal(float64(scalarData.Value))) + Expect(samples[0].Timestamp).To(Equal(int64(scalarData.Timestamp))) + }) + + It("should unmarshal vector responses", func() { + vectorData := model.Vector{ + { + Value: 1, + Timestamp: model.Time(1), + }, + { + Value: 2, + Timestamp: model.Time(0), + }, + } + + dataWrapper := struct { + Type model.ValueType `json:"resultType"` + Result json.RawMessage `json:"result"` + }{} + dataWrapper.Type = model.ValVector + scalarDataBytes, err := json.Marshal(vectorData) + Expect(err).To(Succeed()) + dataWrapper.Result = scalarDataBytes + + scalarDataWrapperBytes, err := json.Marshal(dataWrapper) + Expect(err).To(Succeed()) + + promResp := compat.ApiResponse{ + Status: "success", + Data: scalarDataWrapperBytes, + ErrorType: "", + Error: "", + Warnings: []string{}, + } + + promRespData, err := json.Marshal(promResp) + Expect(err).To(Succeed()) + + qr, err := compat.UnmarshalPrometheusResponse(promRespData) + Expect(err).To(Succeed()) + Expect(qr.Type).To(Equal(model.ValVector)) + + scalar, err := qr.GetScalar() + Expect(err).NotTo(Succeed()) + Expect(scalar).To(BeNil()) + + matrix, err := qr.GetMatrix() + Expect(err).NotTo(Succeed()) + Expect(matrix).To(BeNil()) + + vector, err := qr.GetVector() + Expect(err).To(Succeed()) + Expect(vector).NotTo(BeNil()) + + samples := qr.LinearSamples() + Expect(samples).To(HaveLen(2)) + Expect(samples[0].Value).To(Equal(float64(vectorData[1].Value))) + Expect(samples[0].Timestamp).To(Equal(int64(vectorData[1].Timestamp))) + Expect(samples[1].Value).To(Equal(float64(vectorData[0].Value))) + Expect(samples[1].Timestamp).To(Equal(int64(vectorData[0].Timestamp))) + }) + + It("should unmarshal matrix responses", func() { + matrixData := model.Matrix{ + { + Values: []model.SamplePair{ + { + Value: 1, + Timestamp: model.Time(2), + }, + + { + Value: 2, + Timestamp: model.Time(0), + }, + }, + }, + { + Values: []model.SamplePair{ + { + Value: 3, + Timestamp: model.Time(1), + }, + }, + }, + } + + dataWrapper := struct { + Type model.ValueType `json:"resultType"` + Result json.RawMessage `json:"result"` + }{} + dataWrapper.Type = model.ValMatrix + scalarDataBytes, err := json.Marshal(matrixData) + Expect(err).To(Succeed()) + dataWrapper.Result = scalarDataBytes + + scalarDataWrapperBytes, err := json.Marshal(dataWrapper) + Expect(err).To(Succeed()) + + promResp := compat.ApiResponse{ + Status: "success", + Data: scalarDataWrapperBytes, + ErrorType: "", + Error: "", + Warnings: []string{}, + } + + promRespData, err := json.Marshal(promResp) + Expect(err).To(Succeed()) + + qr, err := compat.UnmarshalPrometheusResponse(promRespData) + Expect(err).To(Succeed()) + Expect(qr.Type).To(Equal(model.ValMatrix)) + + scalar, err := qr.GetScalar() + Expect(err).NotTo(Succeed()) + Expect(scalar).To(BeNil()) + + vector, err := qr.GetVector() + Expect(err).NotTo(Succeed()) + Expect(vector).To(BeNil()) + + matrix, err := qr.GetMatrix() + Expect(err).To(Succeed()) + Expect(matrix).NotTo(BeNil()) + + samples := qr.LinearSamples() + Expect(samples).To(HaveLen(3)) + Expect(samples[0].Value).To(Equal(float64(2))) + Expect(samples[0].Timestamp).To(Equal(int64(0))) + Expect(samples[1].Value).To(Equal(float64(3))) + Expect(samples[1].Timestamp).To(Equal(int64(1))) + Expect(samples[2].Value).To(Equal(float64(1))) + Expect(samples[2].Timestamp).To(Equal(int64(2))) + }) +}) diff --git a/pkg/slo/backend/backend_suite_test.go b/pkg/slo/backend/backend_suite_test.go new file mode 100644 index 0000000000..4948312ed9 --- /dev/null +++ b/pkg/slo/backend/backend_suite_test.go @@ -0,0 +1,14 @@ +package backend_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + _ "github.com/rancher/opni/pkg/test/setup" +) + +func TestBackend(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Backend Suite") +} diff --git a/pkg/slo/backend/event_test.go b/pkg/slo/backend/event_test.go new file mode 100644 index 0000000000..baddfc669d --- /dev/null +++ b/pkg/slo/backend/event_test.go @@ -0,0 +1,122 @@ +package backend_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/slo/backend" +) + +var _ = Describe("Event matching", Label("unit"), func() { + When("we receive event data from the user", func() { + It("should match empty total events to all events", func() { + goodEvents := []*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + } + totalEvents := []*slov1.Event{} + g, t := backend.MatchEventsOnMetric(goodEvents, totalEvents) + Expect(g).To(Equal([]*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + })) + Expect(t).To(Equal([]*slov1.Event{})) + }) + + It("should include good label filters in matching total event filters ", func() { + goodEvents := []*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + } + totalEvents := []*slov1.Event{ + { + Key: "code", + Vals: []string{"500", "503"}, + }, + } + g, t := backend.MatchEventsOnMetric(goodEvents, totalEvents) + Expect(g).To(Equal( + []*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + }, + )) + Expect(t).To(Equal( + []*slov1.Event{ + { + Key: "code", + Vals: []string{"200", "500", "503"}, + }, + }, + )) + }) + + It("should apply missing total label filters to good filters", func() { + goodEvents := []*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + } + totalEvents := []*slov1.Event{ + { + Key: "handler", + Vals: []string{"/ready"}, + }, + } + g, t := backend.MatchEventsOnMetric(goodEvents, totalEvents) + + Expect(g).To(Equal( + []*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + { + Key: "handler", + Vals: []string{"/ready"}, + }, + }, + )) + Expect(t).To(Equal( + []*slov1.Event{ + { + Key: "handler", + Vals: []string{"/ready"}, + }, + }, + )) + }) + + It("should sanitize invalid inputs by omitting them", func() { + goodEvents := []*slov1.Event{ + { + Key: "", + Vals: []string{"200"}, + }, + } + totalEvents := []*slov1.Event{ + { + Key: "", + Vals: []string{"500", "503"}, + }, + { + Key: "handler", + Vals: nil, + }, + } + + g, t := backend.MatchEventsOnMetric(goodEvents, totalEvents) + Expect(g).To(Equal([]*slov1.Event{})) + Expect(t).To(Equal([]*slov1.Event{})) + }) + }) +}) diff --git a/pkg/slo/backend/metrics/backend.go b/pkg/slo/backend/metrics/backend.go new file mode 100644 index 0000000000..3cb176efc2 --- /dev/null +++ b/pkg/slo/backend/metrics/backend.go @@ -0,0 +1,114 @@ +package metrics + +import ( + "context" + "fmt" + + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/metrics/compat" + "github.com/rancher/opni/pkg/slo/backend" + "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" + "github.com/tidwall/gjson" +) + +type MetricsBackend struct { + *MetricsBackendProvider +} + +func NewBackend(p *MetricsBackendProvider) *MetricsBackend { + return &MetricsBackend{ + MetricsBackendProvider: p, + } +} + +var _ backend.ServiceBackend = &MetricsBackend{} + +func (m *MetricsBackend) ListServices(ctx context.Context, req *slov1.ListServicesRequest) (*slov1.ServiceList, error) { + services := &slov1.ServiceList{} + // we should move this to a more generic label discovery API + discoveryQuery := `group by (job) ({__name__!=""})` + resp, err := m.adminClient.Get().Query( + ctx, + &cortexadmin.QueryRequest{ + Tenants: []string{req.GetClusterId()}, + Query: discoveryQuery, + }) + if err != nil { + return nil, err + } + + qr, err := compat.UnmarshalPrometheusResponse(resp.Data) + if err != nil { + return nil, err + } + + vec, err := qr.GetVector() + if err != nil { + return nil, err + } + if vec == nil { + return nil, fmt.Errorf("could not unmarshal cortex query response to json : expeted model.Vector") + } + for _, vecSample := range *vec { + metric := vecSample.Metric + job, ok := metric["job"] + if !ok { + continue + } + services.Items = append(services.Items, &slov1.Service{ + ClusterId: req.GetClusterId(), + ServiceId: string(job), + }) + } + + result := gjson.Get(string(resp.Data), "data.result.#.metric.job") + if !result.Exists() { + return nil, fmt.Errorf("could not unmarshal cortex query response to expected json") + } + for _, v := range result.Array() { + services.Items = append(services.Items, &slov1.Service{ + ClusterId: req.GetClusterId(), + ServiceId: v.String(), + }) + } + return services, nil +} +func (m *MetricsBackend) ListMetrics(ctx context.Context, req *slov1.ListMetricsRequest) (*slov1.MetricGroupList, error) { + resp, err := m.adminClient.Get().GetSeriesMetrics( + ctx, + &cortexadmin.SeriesRequest{ + Tenant: req.GetClusterId(), + JobId: req.GetServiceId(), + }, + ) + if err != nil { + return nil, err + } + return scoredLabels(resp, m.filters), nil +} + +func (m *MetricsBackend) ListEvents(ctx context.Context, req *slov1.ListEventsRequest) (*slov1.EventList, error) { + events := &slov1.EventList{ + Items: []*slov1.Event{}, + } + + resp, err := m.adminClient.Get().GetMetricLabelSets( + ctx, + &cortexadmin.LabelRequest{ + Tenant: req.GetClusterId(), + JobId: req.GetServiceId(), + MetricName: req.GetMetricId(), + }, + ) + if err != nil { + return nil, err + } + for _, item := range resp.GetItems() { + events.Items = append(events.Items, &slov1.Event{ + Key: item.GetName(), + Vals: item.GetItems(), + }) + } + + return events, nil +} diff --git a/pkg/slo/backend/metrics/backend_provider.go b/pkg/slo/backend/metrics/backend_provider.go new file mode 100644 index 0000000000..6a62a87e49 --- /dev/null +++ b/pkg/slo/backend/metrics/backend_provider.go @@ -0,0 +1,184 @@ +package metrics + +import ( + "embed" + "io/fs" + "path/filepath" + "regexp" + "sync" + + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/util/future" + "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" + "github.com/samber/lo" + "go.uber.org/zap" + "gopkg.in/yaml.v3" +) + +//go:embed metricgroups/*.yaml +var metricGroups embed.FS + +//go:embed servicegroups/*.yaml +var serviceGroups embed.FS + +// map of directory names to their embed.FS +var EnabledFilters = map[string]embed.FS{"metricgroups": metricGroups, "servicegroups": serviceGroups} + +type MetricsBackendProvider struct { + lg *zap.SugaredLogger + filters []Filter + adminClient future.Future[cortexadmin.CortexAdminClient] +} + +func NewProvider( + lg *zap.SugaredLogger, +) *MetricsBackendProvider { + return &MetricsBackendProvider{ + lg: lg, + adminClient: future.New[cortexadmin.CortexAdminClient](), + } +} + +func (m *MetricsBackendProvider) Initialize(cl cortexadmin.CortexAdminClient) { + m.adminClient.Set(cl) + m.filters = m.constructFilters() +} + +func ApplyFiltersToMetricEvents(seriesInfo *cortexadmin.SeriesInfoList, filters []Filter) *slov1.MetricGroupList { + return scoredLabels(seriesInfo, filters) +} + +func GetGroupConfigsFromEmbed(dirName string, dir embed.FS) []Filter { + var res []Filter + var mu sync.Mutex + fsys := fs.FS(dir) + yamlFs, err := fs.Sub(fsys, dirName) + if err != nil { + return []Filter{} + } + fs.WalkDir(yamlFs, ".", func(pathStr string, d fs.DirEntry, err error) error { + if !d.IsDir() { + fBytes, err := fs.ReadFile(dir, filepath.Join(dirName, pathStr)) + if err != nil { + panic(err) + } + var f Filter + err = yaml.Unmarshal(fBytes, &f) + if err == nil { + mu.Lock() + res = append(res, f) + mu.Unlock() + } + } + return nil + }) + return res +} + +func scoredLabels(seriesInfo *cortexadmin.SeriesInfoList, filters []Filter) *slov1.MetricGroupList { + + seriesNameToScoreGroup := map[string]map[string]int{} + + seriesNameToSeries := lo.Associate(seriesInfo.GetItems(), func(series *cortexadmin.SeriesInfo) (string, *cortexadmin.SeriesInfo) { + return series.GetSeriesName(), series + }) + + for _, series := range seriesInfo.GetItems() { + seriesName := series.GetSeriesName() + for _, groupname := range filters { + for _, matchFilter := range groupname.Filters { + if matchFilter.Value.MatchString(series.GetSeriesName()) { + if _, ok := seriesNameToScoreGroup[seriesName]; !ok { + seriesNameToScoreGroup[seriesName] = map[string]int{} + } + seriesNameToScoreGroup[seriesName][groupname.Name] += matchFilter.Score + } + } + + for _, ignoreFilter := range groupname.Ignore { + if ignoreFilter.Value.MatchString(series.GetSeriesName()) { + if _, ok := seriesNameToScoreGroup[seriesName]; !ok { + seriesNameToScoreGroup[seriesName] = map[string]int{} + } + seriesNameToScoreGroup[seriesName][groupname.Name] -= ignoreFilter.Score + } + } + } + } + groupedMetrics := &slov1.MetricGroupList{ + GroupNameToMetrics: map[string]*slov1.MetricList{}, + } + for seriesName, data := range seriesNameToSeries { + groupName := "" + maxScore := 0 + + if vals, ok := seriesNameToScoreGroup[seriesName]; ok { + for group, score := range vals { + if score > maxScore { + maxScore = score + groupName = group + } + } + } + + if maxScore <= 0 { + groupName = "other metrics" + } + if _, ok := groupedMetrics.GroupNameToMetrics[groupName]; !ok { + groupedMetrics.GroupNameToMetrics[groupName] = &slov1.MetricList{} + } + groupedMetrics.GroupNameToMetrics[groupName].Items = append(groupedMetrics.GroupNameToMetrics[groupName].Items, &slov1.Metric{ + Id: seriesName, + Metadata: &slov1.MetricMetadata{ + Description: data.Metadata.GetDescription(), + Unit: data.Metadata.GetUnit(), + Type: data.Metadata.GetType(), + }, + }) + } + return groupedMetrics +} + +func (m *MetricsBackendProvider) constructFilters() []Filter { + res := []Filter{} + for dirName, embedFs := range EnabledFilters { + filters := GetGroupConfigsFromEmbed(dirName, embedFs) + res = append(res, filters...) + } + return res +} + +type Filter struct { + Name string `yaml:"name"` + Filters []FilterValue `yaml:"filters"` + Ignore []FilterValue `yaml:"ignore"` +} + +type FilterValue struct { + Value Regexp `yaml:"value"` + Score int `yaml:"score"` +} + +type Regexp struct { + *regexp.Regexp +} + +// UnmarshalText unmarshals json into a regexp.Regexp +func (r *Regexp) UnmarshalText(b []byte) error { + regex, err := regexp.Compile(string(b)) + if err != nil { + return err + } + + r.Regexp = regex + + return nil +} + +// MarshalText marshals regexp.Regexp as string +func (r *Regexp) MarshalText() ([]byte, error) { + if r.Regexp != nil { + return []byte(r.Regexp.String()), nil + } + return nil, nil +} diff --git a/pkg/slo/backend/metrics/backend_test.go b/pkg/slo/backend/metrics/backend_test.go new file mode 100644 index 0000000000..e1f18e5471 --- /dev/null +++ b/pkg/slo/backend/metrics/backend_test.go @@ -0,0 +1,150 @@ +package metrics_test + +import ( + "context" + "encoding/json" + + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/common/model" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/logger" + "github.com/rancher/opni/pkg/metrics/compat" + "github.com/rancher/opni/pkg/slo/backend/metrics" + cortexadmin_mock "github.com/rancher/opni/pkg/test/mock/cortexadmin" + "github.com/rancher/opni/pkg/test/testutil" + "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" + "google.golang.org/grpc" +) + +var _ = Describe("Metrics SLO Service backend", Label("unit"), Ordered, func() { + var mb *metrics.MetricsBackend + BeforeAll(func() { + ctrl := gomock.NewController(GinkgoT()) + mockClient := cortexadmin_mock.NewMockCortexAdminClient(ctrl) + mockClient.EXPECT(). + Query(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, req *cortexadmin.QueryRequest, _ ...grpc.CallOption) (*cortexadmin.QueryResponse, error) { + v := model.Vector{ + { + Metric: model.Metric{ + "job": "test-service", + }, + Value: 1, + Timestamp: model.Time(100000), + }, + } + + wrapper := struct { + Type model.ValueType `json:"resultType"` + Result json.RawMessage `json:"result"` + }{} + wrapper.Type = model.ValVector + wrapperBytes, err := json.Marshal(v) + if err != nil { + return nil, nil + } + wrapper.Result = wrapperBytes + + vectorData, err := json.Marshal(wrapper) + if err != nil { + return nil, err + } + + promResp := compat.ApiResponse{ + Status: "success", + Data: vectorData, + ErrorType: "", + Error: "", + Warnings: []string{}, + } + data, err := json.Marshal(promResp) + if err != nil { + return nil, err + } + return &cortexadmin.QueryResponse{ + Data: data, + }, nil + }).AnyTimes() + + mockClient.EXPECT(). + GetSeriesMetrics(gomock.Any(), gomock.Any()). + Return(&cortexadmin.SeriesInfoList{ + Items: []*cortexadmin.SeriesInfo{ + { + SeriesName: "test", + }, + }, + }, nil).AnyTimes() + + mockClient.EXPECT(). + GetMetricLabelSets(gomock.Any(), gomock.Any()). + Return(&cortexadmin.MetricLabels{ + Items: []*cortexadmin.LabelSet{ + { + Name: "test-label", + Items: []string{"host_name"}, + }, + }, + }, nil).AnyTimes() + + m := metrics.NewProvider(logger.NewPluginLogger().Named("slo")) + m.Initialize(mockClient) + mb = metrics.NewBackend(m) + }) + When("we we use the metrics service backend", func() { + It("should list discoverable services", func() { + svcList, err := mb.ListServices( + context.TODO(), + &slov1.ListServicesRequest{ + Datasource: "metrics", + ClusterId: "test", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(svcList.Items).To(ContainElement(&slov1.Service{ + ClusterId: "test", + ServiceId: "test-service", + })) + }) + + It("should list metrics based on services", func() { + metricList, err := mb.ListMetrics( + context.TODO(), + &slov1.ListMetricsRequest{ + Datasource: "metrics", + ClusterId: "test", + ServiceId: "test", + }, + ) + Expect(err).NotTo(HaveOccurred()) + expected := &slov1.MetricGroupList{ + GroupNameToMetrics: map[string]*slov1.MetricList{ + "other metrics": { + Items: []*slov1.Metric{{Id: "test", Metadata: &slov1.MetricMetadata{}}}, + }, + }, + } + Expect(metricList).To(testutil.ProtoEqual(expected)) + }) + + It("should list events based on metrics and services", func() { + eventList, err := mb.ListEvents(context.TODO(), + &slov1.ListEventsRequest{ + Datasource: "metrics", + ClusterId: "test", + ServiceId: "test", + MetricId: "test", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(eventList).To(Equal(&slov1.EventList{ + Items: []*slov1.Event{ + { + Key: "test-label", + Vals: []string{"host_name"}, + }, + }, + })) + }) + }) +}) diff --git a/pkg/slo/backend/metrics/cortex.go b/pkg/slo/backend/metrics/cortex.go new file mode 100644 index 0000000000..2759a187c4 --- /dev/null +++ b/pkg/slo/backend/metrics/cortex.go @@ -0,0 +1,9 @@ +package metrics + +import ( + "github.com/prometheus/prometheus/model/rulefmt" +) + +type RuleGroupBuilder interface { + AsRuleGroup() (rulefmt.RuleGroup, error) +} diff --git a/pkg/slo/backend/metrics/filter_test.go b/pkg/slo/backend/metrics/filter_test.go new file mode 100644 index 0000000000..d04c376201 --- /dev/null +++ b/pkg/slo/backend/metrics/filter_test.go @@ -0,0 +1,73 @@ +package metrics_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/rancher/opni/pkg/slo/backend/metrics" + "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" +) + +var _ = Describe("SLO Filter tests", Ordered, Label("unit"), func() { + When("We use SLO filters", func() { + var filters []metrics.Filter + It("should get parse them from our embedded directory definitions", func() { + filters = make([]metrics.Filter, 0) + for dirName, embedFs := range metrics.EnabledFilters { + flts := metrics.GetGroupConfigsFromEmbed(dirName, embedFs) + filters = append(filters, flts...) + } + for _, filter := range filters { + Expect(filter.Name).NotTo(Equal("")) + Expect(filter.Filters).NotTo(BeEmpty()) + } + Expect(filters).To(HaveLen(6)) + }) + + It("Should be able to score the events based on the filters", func() { + // array of prom metrics : label name -> label vals + series := &cortexadmin.SeriesInfoList{ + Items: []*cortexadmin.SeriesInfo{ + { + SeriesName: "go_gc_duration_seconds", + }, + { + SeriesName: "jvm_something", + }, + { + SeriesName: "jvm_something_else", + }, + { + SeriesName: "uptime_seconds", + }, + { + SeriesName: "kube-proxy", + }, + { + SeriesName: "apiserver_something", + }, + { + SeriesName: "request_duration_seconds", + }, + { + SeriesName: "cpu_usage_seconds_total", + }, + { + SeriesName: ".*kube-prometheus-operator.*", + }, + }, + } + + filteredGroupsResp := metrics.ApplyFiltersToMetricEvents(series, filters) + Expect(filteredGroupsResp).NotTo(BeNil()) + + filteredGroups := filteredGroupsResp.GetGroupNameToMetrics() + + Expect(filteredGroups).To(HaveKey("golang metrics")) + Expect(filteredGroups).To(HaveKey("jvm metrics")) + Expect(filteredGroups).To(HaveKey("kubernetes metrics")) + Expect(filteredGroups).To(HaveKey("network metrics")) + Expect(filteredGroups).To(HaveKey("compute metrics")) + Expect(filteredGroups).To(HaveKey("other metrics")) + }) + }) +}) diff --git a/pkg/slo/backend/metrics/manager.go b/pkg/slo/backend/metrics/manager.go new file mode 100644 index 0000000000..f614549628 --- /dev/null +++ b/pkg/slo/backend/metrics/manager.go @@ -0,0 +1,435 @@ +package metrics + +import ( + "context" + "sync" + "time" + + "github.com/google/uuid" + prommodel "github.com/prometheus/common/model" + corev1 "github.com/rancher/opni/pkg/apis/core/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/metrics/compat" + "github.com/rancher/opni/pkg/slo/backend" + "github.com/rancher/opni/pkg/util" + "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" + "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" + "gopkg.in/yaml.v3" +) + +const ( + sloRuleNamespace = "slo" +) + +type MetricsSLOStoreProvider struct { + lg *zap.SugaredLogger + adminClient cortexadmin.CortexAdminClient +} + +func NewMetricsSLOStoreProvider( + lg *zap.SugaredLogger, +) *MetricsSLOStoreProvider { + return &MetricsSLOStoreProvider{ + lg: lg, + } +} + +func (m *MetricsSLOStoreProvider) Initialize(adminClient cortexadmin.CortexAdminClient) { + m.adminClient = adminClient +} + +type MetricsSLOStore struct { + *MetricsSLOStoreProvider + + // backend.ServiceBackend +} + +var _ backend.SLOStore = (*MetricsSLOStore)(nil) + +func NewMetricsSLOStore( + provider *MetricsSLOStoreProvider, +) *MetricsSLOStore { + return &MetricsSLOStore{ + MetricsSLOStoreProvider: provider, + } +} + +func (m *MetricsSLOStore) Create(ctx context.Context, req *slov1.CreateSLORequest) (*corev1.Reference, error) { + slo := backend.CreateSLORequestToStruct(req) + sloGen, err := NewSLOGenerator(*slo) + if err != nil { + return nil, err + } + ruleGroup, err := sloGen.AsRuleGroup() + if err != nil { + return nil, err + } + yamlBytes, err := yaml.Marshal(ruleGroup) + if err != nil { + return nil, err + } + if _, err := m.adminClient.LoadRules(ctx, &cortexadmin.LoadRuleRequest{ + Namespace: sloRuleNamespace, + YamlContent: yamlBytes, + ClusterId: req.Slo.ClusterId, + }); err != nil { + return nil, err + } + + return &corev1.Reference{ + Id: slo.GetId(), + }, nil +} +func (m *MetricsSLOStore) Update(ctx context.Context, incoming, existing *slov1.SLOData) (*slov1.SLOData, error) { + incomingSLO := backend.SLODataToStruct(incoming) + existingSLO := backend.SLODataToStruct(existing) + if err := incomingSLO.Validate(); err != nil { + return nil, err + } + if err := existingSLO.Validate(); err != nil { + return nil, err + } + sloGen, err := NewSLOGenerator(*incomingSLO) + if err != nil { + return nil, err + } + rg, err := sloGen.AsRuleGroup() + if err != nil { + return nil, err + } + + yamlBytes, err := yaml.Marshal(rg) + if err != nil { + return nil, err + } + if _, err := m.adminClient.LoadRules(ctx, &cortexadmin.LoadRuleRequest{ + Namespace: sloRuleNamespace, + YamlContent: yamlBytes, + ClusterId: incoming.SLO.ClusterId, + }); err != nil { + return nil, err + } + + if incoming.SLO.ClusterId != existing.SLO.ClusterId { + m.adminClient.DeleteRule(ctx, &cortexadmin.DeleteRuleRequest{ + Namespace: sloRuleNamespace, + GroupName: existingSLO.GetId(), + ClusterId: existing.SLO.ClusterId, + }) + } + return incoming, nil +} +func (m *MetricsSLOStore) Delete(ctx context.Context, existing *slov1.SLOData) error { + slo := backend.SLODataToStruct(existing) + _, err := m.adminClient.DeleteRule(ctx, &cortexadmin.DeleteRuleRequest{ + Namespace: sloRuleNamespace, + GroupName: slo.GetId(), + ClusterId: existing.SLO.ClusterId, + }) + return err +} +func (m *MetricsSLOStore) Clone(ctx context.Context, clone *slov1.SLOData) (*corev1.Reference, *slov1.SLOData, error) { + clonedData := util.ProtoClone(clone) + sloData := clonedData.GetSLO() + clonedSLO := backend.SLODataToStruct(clonedData) + clonedSLO.SetId(uuid.New().String()) + clonedSLO.SetName(sloData.GetName() + "-clone") + yamlBytes, err := yaml.Marshal(clonedSLO) + if err != nil { + return nil, nil, err + } + if _, err := m.adminClient.LoadRules(ctx, &cortexadmin.LoadRuleRequest{ + Namespace: sloRuleNamespace, + YamlContent: yamlBytes, + ClusterId: sloData.ClusterId, + }); err != nil { + return nil, nil, err + } + return &corev1.Reference{ + Id: clonedSLO.GetId(), + }, clonedData, nil +} +func (m *MetricsSLOStore) MultiClusterClone( + ctx context.Context, + slo *slov1.SLOData, + toClusters []*corev1.Reference, +) ([]*corev1.Reference, []*slov1.SLOData, []error) { + errors := make([]error, len(toClusters)) + cloned := make([]*slov1.SLOData, len(toClusters)) + refs := make([]*corev1.Reference, len(toClusters)) + var wg sync.WaitGroup + var mu sync.Mutex + for i, cluster := range toClusters { + cluster := cluster + i := i + wg.Add(1) + go func() { + defer wg.Done() + clone, _, err := m.Clone(ctx, slo) + mu.Lock() + defer mu.Unlock() + errors[i] = err + if err != nil { + return + } + cloned[i] = util.ProtoClone(slo) + cloned[i].SLO.ClusterId = cluster.Id + refs[i] = clone + }() + } + wg.Wait() + return refs, cloned, errors + +} + +// 1. Check if the rule is loaded +// 2. Check SLI, if there is no data, return NoData +// 3. Check the error budget remaining, if exceeded, return Breaching +// 4. Check the MWMB burn rates, if they are elevated, return Warning +// 5. Otherwise, return Ok +func (m *MetricsSLOStore) Status(ctx context.Context, slo *slov1.SLOData) (*slov1.SLOStatus, error) { + clusterId := slo.SLO.ClusterId + // 1. + _, err := m.adminClient.GetRule(ctx, &cortexadmin.GetRuleRequest{ + Namespace: sloRuleNamespace, + ClusterId: clusterId, + GroupName: slo.GetId(), + }) + if st, ok := status.FromError(err); ok && st.Code() == codes.NotFound { + return &slov1.SLOStatus{ + State: slov1.SLOStatusState_Creating, + }, nil + } else if err != nil { + return nil, err + } + + sloGen, err := NewSLOGenerator(*backend.SLODataToStruct(slo)) + if err != nil { + return nil, err + } + + // 2. + dur, err := prommodel.ParseDuration(slo.SLO.SloPeriod) + if err != nil { + return nil, err + } + sliExpr := sloGen.SLI(WindowMetadata{ + WindowDur: dur, + Name: "test", + }).Expr() + rawSLIRes, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + Tenants: []string{clusterId}, + Query: sliExpr, + }) + if err != nil { + return nil, err + } + qrSLI, err := compat.UnmarshalPrometheusResponse(rawSLIRes.Data) + if err != nil { + return nil, err + } + if len(qrSLI.LinearSamples()) == 0 { + return &slov1.SLOStatus{ + State: slov1.SLOStatusState_NoData, + }, nil + } + + // 3. + remainingBudget := sloGen.ErrorBudgetRemaining().Expr() + rawRemBudget, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + Tenants: []string{clusterId}, + Query: remainingBudget, + }) + if err != nil { + return nil, err + } + qrBudget, err := compat.UnmarshalPrometheusResponse(rawRemBudget.Data) + if err != nil { + return nil, err + } + samples := qrBudget.LinearSamples() + if len(samples) == 0 { + return &slov1.SLOStatus{ + State: slov1.SLOStatusState_NoData, + }, nil + } + lastErrorBudget := samples[len(samples)-1].Value + if lastErrorBudget < 0 { + return &slov1.SLOStatus{ + State: slov1.SLOStatusState_Breaching, + }, nil + } + + ticketAlert := sloGen.TicketAlert().Expr() + rawTicketAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + Tenants: []string{clusterId}, + Query: ticketAlert, + }) + if err != nil { + return nil, err + } + qrTicket, err := compat.UnmarshalPrometheusResponse(rawTicketAlert.Data) + if err != nil { + return nil, err + } + samples = qrTicket.LinearSamples() + if len(samples) != 0 { + lastPageAlert := samples[len(samples)-1].Value + if lastPageAlert > 0 { + return &slov1.SLOStatus{ + State: slov1.SLOStatusState_Warning, + }, nil + } + } + + // 4. + pageAlert := sloGen.PageAlert().Expr() + rawPageAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + Tenants: []string{clusterId}, + Query: pageAlert, + }) + if err != nil { + return nil, err + } + qrPage, err := compat.UnmarshalPrometheusResponse(rawPageAlert.Data) + if err != nil { + return nil, err + } + samples = qrPage.LinearSamples() + if len(samples) != 0 { + lastPageAlert := samples[len(samples)-1].Value + if lastPageAlert > 0 { + return &slov1.SLOStatus{ + State: slov1.SLOStatusState_Warning, + }, nil + } + } + + return &slov1.SLOStatus{ + State: slov1.SLOStatusState_Ok, + }, nil +} +func (m *MetricsSLOStore) Preview(ctx context.Context, slo *slov1.CreateSLORequest) (*slov1.SLOPreviewResponse, error) { + sloDef := backend.SLODataToStruct(&slov1.SLOData{ + Id: "preview-" + uuid.New().String(), + SLO: slo.Slo, + }) + + sloGen, err := NewSLOGenerator(*sloDef) + if err != nil { + return nil, err + } + + period, err := prommodel.ParseDuration(slo.Slo.SloPeriod) + if err != nil { + return nil, err + } + sli := sloGen.SLI(WindowMetadata{WindowDur: period, Name: "test"}).Expr() + cur := time.Now() + startTs, endTs := cur.Add(time.Duration(-period)), cur + numSteps := 250 + step := time.Duration(endTs.Sub(startTs).Seconds()/float64(numSteps)) * time.Second + if step < time.Second { + step = time.Second + } + rawSLI, err := m.adminClient.QueryRange(ctx, &cortexadmin.QueryRangeRequest{ + Tenants: []string{slo.Slo.ClusterId}, + Query: sli, + Start: timestamppb.New(startTs), + End: timestamppb.New(endTs), + Step: durationpb.New(step), + }) + if err != nil { + return nil, err + } + + qrSLI, err := compat.UnmarshalPrometheusResponse(rawSLI.Data) + if err != nil { + return nil, err + } + + samples := qrSLI.LinearSamples() + + plotVector := &slov1.PlotVector{ + Objective: slo.Slo.Target.Value, + Items: make([]*slov1.DataPoint, len(samples)), + } + + for i := 0; i < len(samples); i++ { + plotVector.Items[i] = &slov1.DataPoint{ + Sli: samples[i].Value * 100, + Timestamp: timestamppb.Now(), + } + } + + pageIntervals := sloGen.PageIntervals().Expr() + rawPageAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + Tenants: []string{slo.Slo.ClusterId}, + Query: pageIntervals, + }) + if err != nil { + return nil, err + } + + qrPage, err := compat.UnmarshalPrometheusResponse(rawPageAlert.Data) + if err != nil { + return nil, err + } + + pageSamples := qrPage.LinearSamples() + plotVector.Windows = append(plotVector.Windows, DetectFiringIntervals(severeAlertWindow, pageSamples)...) + + ticketIntervals := sloGen.TicketIntervals().Expr() + rawTicketAlert, err := m.adminClient.Query(ctx, &cortexadmin.QueryRequest{ + Tenants: []string{slo.Slo.ClusterId}, + Query: ticketIntervals, + }) + if err != nil { + return nil, err + } + + qrTicket, err := compat.UnmarshalPrometheusResponse(rawTicketAlert.Data) + if err != nil { + return nil, err + } + + ticketSamples := qrTicket.LinearSamples() + plotVector.Windows = append(plotVector.Windows, DetectFiringIntervals(criticalAlertWindow, ticketSamples)...) + + return &slov1.SLOPreviewResponse{ + PlotVector: plotVector, + }, nil +} + +const severeAlertWindow = "severe" +const criticalAlertWindow = "critical" + +func DetectFiringIntervals(sev string, samples []compat.Sample) []*slov1.AlertFiringWindows { + windows := make([]*slov1.AlertFiringWindows, 0) + open := false + for _, sample := range samples { + if !open && sample.Value > 0 { + // Convert Unix millisecond timestamp to time.Time + t := time.Unix(sample.Timestamp/1000, (sample.Timestamp%1000)*int64(time.Millisecond)) + windows = append(windows, &slov1.AlertFiringWindows{ + Severity: sev, + Start: timestamppb.New(t), + }) + open = true + } else if open && sample.Value == 0 { + t := time.Unix(sample.Timestamp/1000, (sample.Timestamp%1000)*int64(time.Millisecond)) + windows[len(windows)-1].End = timestamppb.New(t) + open = false + } + } + + if open && windows[len(windows)-1].End == nil { + windows[len(windows)-1].End = timestamppb.Now() + } + + return windows +} diff --git a/pkg/slo/backend/metrics/manager_test.go b/pkg/slo/backend/metrics/manager_test.go new file mode 100644 index 0000000000..7379cfeac0 --- /dev/null +++ b/pkg/slo/backend/metrics/manager_test.go @@ -0,0 +1,308 @@ +package metrics_test + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/golang/mock/gomock" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/common/model" + corev1 "github.com/rancher/opni/pkg/apis/core/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/logger" + "github.com/rancher/opni/pkg/metrics/compat" + "github.com/rancher/opni/pkg/slo/backend/metrics" + cortexadmin_mock "github.com/rancher/opni/pkg/test/mock/cortexadmin" + "github.com/rancher/opni/pkg/test/testutil" + "github.com/rancher/opni/pkg/util" + "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +var _ = Describe("SLO Metrics store", Ordered, Label("unit"), func() { + var metricsStore *metrics.MetricsSLOStore + BeforeAll(func() { + ctrl := gomock.NewController(GinkgoT()) + + mockClient := cortexadmin_mock.NewMockCortexAdminClient(ctrl) + + mockClient.EXPECT().LoadRules(gomock.Any(), gomock.Any()).Return(&emptypb.Empty{}, nil).AnyTimes() + + mockClient.EXPECT().DeleteRule(gomock.Any(), gomock.Any()).Return(&emptypb.Empty{}, nil).AnyTimes() + + mockClient.EXPECT().GetRule(gomock.Any(), gomock.Any()).Return(nil, status.Error(codes.NotFound, "rule not found")).AnyTimes() + + mockClient.EXPECT().Query(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, req *cortexadmin.QueryRequest, _ ...grpc.CallOption) (*cortexadmin.QueryResponse, error) { + v := model.Vector{ + { + Metric: model.Metric{ + "job": "test-service", + }, + Value: 1, + Timestamp: model.Time(100000), + }, + } + + wrapper := struct { + Type model.ValueType `json:"resultType"` + Result json.RawMessage `json:"result"` + }{} + wrapper.Type = model.ValVector + wrapperBytes, err := json.Marshal(v) + if err != nil { + return nil, nil + } + wrapper.Result = wrapperBytes + + vectorData, err := json.Marshal(wrapper) + if err != nil { + return nil, err + } + + promResp := compat.ApiResponse{ + Status: "success", + Data: vectorData, + ErrorType: "", + Error: "", + Warnings: []string{}, + } + data, err := json.Marshal(promResp) + if err != nil { + return nil, err + } + return &cortexadmin.QueryResponse{ + Data: data, + }, nil + }, + ).AnyTimes() + + mockClient.EXPECT().QueryRange(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, req *cortexadmin.QueryRangeRequest, _ ...grpc.CallOption) (*cortexadmin.QueryResponse, error) { + v := model.Vector{ + { + Metric: model.Metric{ + "job": "test-service", + }, + Value: 1, + Timestamp: model.Time(100000), + }, + } + + wrapper := struct { + Type model.ValueType `json:"resultType"` + Result json.RawMessage `json:"result"` + }{} + wrapper.Type = model.ValVector + wrapperBytes, err := json.Marshal(v) + if err != nil { + return nil, nil + } + wrapper.Result = wrapperBytes + + vectorData, err := json.Marshal(wrapper) + if err != nil { + return nil, err + } + + promResp := compat.ApiResponse{ + Status: "success", + Data: vectorData, + ErrorType: "", + Error: "", + Warnings: []string{}, + } + data, err := json.Marshal(promResp) + if err != nil { + return nil, err + } + return &cortexadmin.QueryResponse{ + Data: data, + }, nil + }, + ).AnyTimes() + + lg := logger.NewPluginLogger().Named("slo") + provider := metrics.NewMetricsSLOStoreProvider(lg) + provider.Initialize(mockClient) + metricsStore = metrics.NewMetricsSLOStore( + provider, + ) + }) + + It("should detect active SLO windows from samples", func() { + samples := []compat.Sample{ + { + Value: 0, + Timestamp: 1, + }, + { + Value: 5, + Timestamp: 2, + }, + { + Value: 10, + Timestamp: 10, + }, + { + Value: 0, + Timestamp: 11, + }, + { + Value: 0, + Timestamp: 15, + }, + { + Value: 1, + Timestamp: 33, + }, + { + Value: 0, + Timestamp: 34, + }, + { + Value: 55, + Timestamp: 4000, + }, + } + windows := metrics.DetectFiringIntervals("critical", samples) + + convert := func(a int64) time.Time { + return time.Unix(a/1000, (a%1000)*int64(time.Millisecond)) + } + + // t := time.Unix(sample.Timestamp/1000, (sample.Timestamp%1000)*int64(time.Millisecond)) + first := &slov1.AlertFiringWindows{Start: timestamppb.New(convert(2)), End: timestamppb.New(convert(11)), Severity: "critical"} + second := &slov1.AlertFiringWindows{Start: timestamppb.New(convert(33)), End: timestamppb.New(convert(34)), Severity: "critical"} + third := &slov1.AlertFiringWindows{Start: timestamppb.New(convert(4000)), End: timestamppb.New(convert(4000)), Severity: "critical"} + + Expect(len(windows)).To(Equal(3)) + Expect(windows[0]).To(testutil.ProtoEqual(first)) + Expect(windows[1]).To(testutil.ProtoEqual(second)) + last := windows[2] + Expect(last.Start).To(testutil.ProtoEqual(third.Start)) + Expect(last.Severity).To(Equal(third.Severity)) + Expect(last.End).NotTo(BeNil()) + Expect(last.End.AsTime().After(time.Now().Add(-time.Minute))).To(BeTrue()) + + }) + + When("we use the mocked metrics store", func() { + It("should be able to create metrics SLOs", func() { + ref, err := metricsStore.Create(context.TODO(), &slov1.CreateSLORequest{ + Slo: util.ProtoClone(mockSLO), + }) + Expect(err).To(Succeed()) + Expect(ref).NotTo(BeNil()) + }) + + It("should be able to update metrics SLOs", func() { + ref, err := metricsStore.Update( + context.TODO(), + &slov1.SLOData{ + Id: uuid.New().String(), + SLO: util.ProtoClone(mockSLO), + }, + &slov1.SLOData{ + Id: uuid.New().String(), + SLO: util.ProtoClone(mockSLO), + }, + ) + Expect(err).To(Succeed()) + Expect(ref).NotTo(BeNil()) + }) + + It("should be able to clone metrics SLOs", func() { + ref, _, err := metricsStore.Clone( + context.TODO(), + &slov1.SLOData{ + Id: uuid.New().String(), + SLO: util.ProtoClone(mockSLO), + }, + ) + Expect(err).To(Succeed()) + Expect(ref).NotTo(BeNil()) + }) + + It("should be able to clone metrics SLOs across clusters", func() { + refs, _, errs := metricsStore.MultiClusterClone( + context.TODO(), + &slov1.SLOData{ + Id: uuid.New().String(), + SLO: util.ProtoClone(mockSLO), + }, + []*corev1.Reference{ + { + Id: "test", + }, + }, + ) + Expect(errors.Join(errs...)).To(Succeed()) + for _, ref := range refs { + Expect(ref).NotTo(BeNil()) + } + }) + + It("should be able to delete metrics SLOs", func() { + err := metricsStore.Delete( + context.TODO(), + &slov1.SLOData{ + Id: uuid.New().String(), + SLO: util.ProtoClone(mockSLO), + }, + ) + Expect(err).To(Succeed()) + }) + + It("should be able to fetch the status of metrics SLOs", func() { + _, err := metricsStore.Status( + context.TODO(), + &slov1.SLOData{ + Id: uuid.New().String(), + SLO: util.ProtoClone(mockSLO), + }, + ) + Expect(err).To(Succeed()) + }) + + It("should fetch previews of the metrics SLOS", func() { + _, err := metricsStore.Preview( + context.TODO(), + &slov1.CreateSLORequest{ + Slo: util.ProtoClone(mockSLO), + }, + ) + Expect(err).To(Succeed()) + }) + }) +}) + +var ( + mockSLO = &slov1.ServiceLevelObjective{ + Name: "test-slo-success", + ClusterId: "agent", + ServiceId: "test-service", + GoodMetricName: "testmetric", + GoodEvents: []*slov1.Event{ + { + Key: "a", + Vals: []string{"a", "b", "c"}, + }, + }, + TotalMetricName: "testmetric", + TotalEvents: []*slov1.Event{}, + Target: &slov1.Target{Value: 99.9}, + SloPeriod: "30d", + BudgetingInterval: durationpb.New(time.Minute * 5), + Datasource: "monitoring", + } +) diff --git a/plugins/slo/pkg/slo/metricgroups/compute.yaml b/pkg/slo/backend/metrics/metricgroups/compute.yaml similarity index 100% rename from plugins/slo/pkg/slo/metricgroups/compute.yaml rename to pkg/slo/backend/metrics/metricgroups/compute.yaml diff --git a/plugins/slo/pkg/slo/metricgroups/go_lang.yaml b/pkg/slo/backend/metrics/metricgroups/go_lang.yaml similarity index 100% rename from plugins/slo/pkg/slo/metricgroups/go_lang.yaml rename to pkg/slo/backend/metrics/metricgroups/go_lang.yaml diff --git a/plugins/slo/pkg/slo/metricgroups/java_lang.yaml b/pkg/slo/backend/metrics/metricgroups/java_lang.yaml similarity index 100% rename from plugins/slo/pkg/slo/metricgroups/java_lang.yaml rename to pkg/slo/backend/metrics/metricgroups/java_lang.yaml diff --git a/plugins/slo/pkg/slo/metricgroups/network.yaml b/pkg/slo/backend/metrics/metricgroups/network.yaml similarity index 100% rename from plugins/slo/pkg/slo/metricgroups/network.yaml rename to pkg/slo/backend/metrics/metricgroups/network.yaml diff --git a/plugins/slo/pkg/slo/metricgroups/other.yaml b/pkg/slo/backend/metrics/metricgroups/other.yaml similarity index 100% rename from plugins/slo/pkg/slo/metricgroups/other.yaml rename to pkg/slo/backend/metrics/metricgroups/other.yaml diff --git a/pkg/slo/backend/metrics/metrics_suite_test.go b/pkg/slo/backend/metrics/metrics_suite_test.go new file mode 100644 index 0000000000..6b56b1be87 --- /dev/null +++ b/pkg/slo/backend/metrics/metrics_suite_test.go @@ -0,0 +1,14 @@ +package metrics_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + _ "github.com/rancher/opni/pkg/test/setup" +) + +func TestMetrics(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Metrics Suite") +} diff --git a/pkg/slo/backend/metrics/prometheus.go b/pkg/slo/backend/metrics/prometheus.go new file mode 100644 index 0000000000..58987f38e0 --- /dev/null +++ b/pkg/slo/backend/metrics/prometheus.go @@ -0,0 +1,650 @@ +package metrics + +import ( + "errors" + "fmt" + "time" + + prommodel "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/rulefmt" + "github.com/rancher/opni/pkg/slo/backend" + "gopkg.in/yaml.v3" +) + +var DefaultEvaluationInterval = time.Minute + +const ( + SLOSLI = "slo:sli_error:ratio_rate" + SLOObjective = "slo:objective:ratio" + SLOErrorBudget = "slo:error_budget:ratio" + SLOPeriod = "slo:time_period:duration" + SLOCurrentBurnRate = "slo:current_burn_rate:ratio_rate" + SLOPeriodBurnRate = "slo:period_burn_rate:ratio_rate" + SLOErrorBudgetRemaining = "slo:error_budget_remaining:ratio" + SLOInfo = "opni_slo_info" +) + +func QueryWithLabels(query string, labels backend.IdentificationLabels) string { + return fmt.Sprintf( + "%s{%s}", + query, + labels.ToLabels().ConstructPrometheus(), + ) +} + +type PrometheusSLO struct { + SLI MetricGenerator +} + +type MetricGenerator interface { + Id() string + Expr() string + Rule() (rulefmt.RuleNode, error) +} + +type MWMBSLOGenerator interface { + Windows() []WindowMetadata + SLI(WindowMetadata) MetricGenerator + + Objective() MetricGenerator + ErrorBudget() MetricGenerator + Period() MetricGenerator + Info() MetricGenerator + + CurrentBurnRate() MetricGenerator + PeriodBurnRate() MetricGenerator + ErrorBudgetRemaining() MetricGenerator + + PageAlert() MetricGenerator + TicketAlert() MetricGenerator + PageIntervals() MetricGenerator + TicketIntervals() MetricGenerator +} + +type SLOGeneratorOptions struct { + optimized bool +} + +func (s *SLOGeneratorOptions) Apply(opts ...SLOGeneratorOption) { + for _, opt := range opts { + opt(s) + } +} + +func DefaultSLOGeneratorOptions() *SLOGeneratorOptions { + return &SLOGeneratorOptions{ + optimized: true, + } +} + +type SLOGeneratorOption func(*SLOGeneratorOptions) + +func WithOptimization(optimized bool) SLOGeneratorOption { + return func(o *SLOGeneratorOptions) { + o.optimized = optimized + } +} + +func NewSLOGenerator(slo backend.SLO, opts ...SLOGeneratorOption) (*SLOGeneratorImpl, error) { + options := DefaultSLOGeneratorOptions() + options.Apply(opts...) + if err := slo.Validate(); err != nil { + return nil, err + } + sloPeriod, err := prommodel.ParseDuration(slo.SloPeriod) + if err != nil { + return nil, err + } + goodQuery := fmt.Sprintf("%s{job=\"%s\", %s}", slo.GoodMetric, slo.Svc, slo.GoodEvents.ConstructPrometheus()) + var totalQuery string + if len(slo.TotalEvents) == 0 { + totalQuery = fmt.Sprintf("%s{job=\"%s\"}", slo.TotalMetric, slo.Svc) + } else { + totalQuery = fmt.Sprintf("%s{job=\"%s\", %s}", slo.TotalMetric, slo.Svc, slo.TotalEvents.ConstructPrometheus()) + } + var windows *MWMBWindows + switch slo.SloPeriod { + case "28d", "30d": //fallback to google SRE production presets + windows = GenerateMWMBWindows( + sloPeriod, + prommodel.Duration(time.Minute*5), + ) + default: // use somewhat hacky normalization + windows = GenerateMWMBWindows( + sloPeriod, + prommodel.Duration(NormalizePeriodToBudgetingInterval(time.Duration(sloPeriod))), + ) + } + + return &SLOGeneratorImpl{ + slo: slo, + options: options, + goodQuery: goodQuery, + totalQuery: totalQuery, + period: sloPeriod, + windows: windows, + }, nil +} + +type SLOGeneratorImpl struct { + slo backend.SLO + + goodQuery string + totalQuery string + period prommodel.Duration + + windows *MWMBWindows + + options *SLOGeneratorOptions +} + +var _ MWMBSLOGenerator = (*SLOGeneratorImpl)(nil) +var _ RuleGroupBuilder = (*SLOGeneratorImpl)(nil) + +func (s *SLOGeneratorImpl) groupId() string { + return "slo-mwmb" +} + +func (s *SLOGeneratorImpl) AsRuleGroup() (rulefmt.RuleGroup, error) { + rules, err := s.rules() + if err != nil { + return rulefmt.RuleGroup{}, err + } + return rulefmt.RuleGroup{ + Name: s.slo.GetId(), + Interval: prommodel.Duration(DefaultEvaluationInterval), + Rules: rules, + }, nil +} + +func (s *SLOGeneratorImpl) rules() ([]rulefmt.RuleNode, error) { + ruleConstructor := []func() (rulefmt.RuleNode, error){ + s.Objective().Rule, + s.ErrorBudget().Rule, + s.Period().Rule, + s.Info().Rule, + s.CurrentBurnRate().Rule, + s.PeriodBurnRate().Rule, + s.ErrorBudgetRemaining().Rule, + } + + for _, window := range s.Windows() { + sli := s.SLI(window) + ruleConstructor = append(ruleConstructor, sli.Rule) + } + + rules := []rulefmt.RuleNode{} + var errs []error + for _, constructor := range ruleConstructor { + rule, err := constructor() + if err != nil { + errs = append(errs, err) + continue + } + rules = append(rules, rule) + } + return rules, errors.Join(errs...) +} + +func (s *SLOGeneratorImpl) Windows() []WindowMetadata { + return s.windows.WindowRange() +} + +func (s *SLOGeneratorImpl) SLI(w WindowMetadata) MetricGenerator { + return NewMWMBSLIGenerator( + SLOSLI, + s.goodQuery, + s.totalQuery, + w, + s.slo.IdLabels, + ) +} + +func normalizeObjective(objective float64) float64 { + return objective / 100 +} + +// ==== constants +func (s *SLOGeneratorImpl) Objective() MetricGenerator { + return NewConstantMetricGenerator(SLOObjective, normalizeObjective(s.slo.Objective), 9, s.slo.IdLabels) +} + +func (s *SLOGeneratorImpl) ErrorBudget() MetricGenerator { + return NewConstantMetricGenerator(SLOErrorBudget, 1-normalizeObjective(s.slo.Objective), 9, s.slo.IdLabels) +} + +func (s *SLOGeneratorImpl) Period() MetricGenerator { + fSeconds := time.Duration(s.period).Seconds() + return NewConstantMetricGenerator(SLOPeriod, fSeconds, 9, s.slo.IdLabels) +} + +func (s *SLOGeneratorImpl) Info() MetricGenerator { + return NewConstantMetricGenerator(SLOInfo, 1, 1, s.slo.IdLabels) +} + +// ==== Dependents on SLI +func (s *SLOGeneratorImpl) CurrentBurnRate() MetricGenerator { + windows := s.Windows() + return NewBurnRateGenerator( + SLOCurrentBurnRate, + s.slo.IdLabels, + s.SLI(windows[0]), + s.ErrorBudget(), + s.options, + ) +} + +func (s *SLOGeneratorImpl) PeriodBurnRate() MetricGenerator { + periodWindow := s.Windows()[len(s.Windows())-1] + return NewBurnRateGenerator( + SLOPeriodBurnRate, + s.slo.IdLabels, + s.SLI(periodWindow), + s.ErrorBudget(), + s.options, + ) +} + +func (s *SLOGeneratorImpl) ErrorBudgetRemaining() MetricGenerator { + return NewErrorBudgetRemainingGenerator( + SLOErrorBudgetRemaining, + s.slo.IdLabels, + s.PeriodBurnRate(), + s.options, + ) +} + +func (s *SLOGeneratorImpl) PageAlert() MetricGenerator { + return NewMWMBAlertGenerator( + s.slo.IdLabels, + s.windows.PageQuick, + s.windows.PageSlow, + func(window WindowMetadata) MetricGenerator { + return s.SLI(window) + }, + s.options, + false, + ) +} + +func (s *SLOGeneratorImpl) TicketAlert() MetricGenerator { + return NewMWMBAlertGenerator( + s.slo.IdLabels, + s.windows.TicketQuick, + s.windows.TicketSlow, + func(window WindowMetadata) MetricGenerator { + return s.SLI(window) + }, + s.options, + false, + ) +} + +func (s *SLOGeneratorImpl) PageIntervals() MetricGenerator { + return NewMWMBAlertGenerator( + s.slo.IdLabels, + s.windows.PageQuick, + s.windows.PageSlow, + func(window WindowMetadata) MetricGenerator { + return s.SLI(window) + }, + s.options, + true, + ) +} + +func (s *SLOGeneratorImpl) TicketIntervals() MetricGenerator { + return NewMWMBAlertGenerator( + s.slo.IdLabels, + s.windows.TicketQuick, + s.windows.TicketSlow, + func(window WindowMetadata) MetricGenerator { + return s.SLI(window) + }, + s.options, + true, + ) +} + +var _ MWMBSLOGenerator = (*SLOGeneratorImpl)(nil) + +type ConstantMetricGenerator struct { + name string + value float64 + precision int + idLabels backend.IdentificationLabels +} + +var _ MetricGenerator = (*ConstantMetricGenerator)(nil) + +func NewConstantMetricGenerator(id string, value float64, precision int, labels map[string]string) *ConstantMetricGenerator { + return &ConstantMetricGenerator{ + name: id, + value: value, + precision: precision, + idLabels: labels, + } +} + +func (v *ConstantMetricGenerator) Id() string { + return v.name +} + +func (v *ConstantMetricGenerator) Expr() string { + precision := fmt.Sprintf("vector(%%.%df)", v.precision) + return fmt.Sprintf(precision, v.value) +} + +func (v *ConstantMetricGenerator) Rule() (rulefmt.RuleNode, error) { + return rulefmt.RuleNode{ + Record: yaml.Node{ + Kind: yaml.ScalarNode, + Value: v.name, + }, + Expr: yaml.Node{ + Kind: yaml.ScalarNode, + Value: v.Expr(), + }, + Labels: v.idLabels, + }, nil +} + +type SumRateGenerator struct { + goodExpr string + window WindowMetadata +} + +var _ MetricGenerator = (*SumRateGenerator)(nil) + +func NewSumRateGenerator( + goodExpr string, + window WindowMetadata, +) *SumRateGenerator { + return &SumRateGenerator{ + goodExpr: goodExpr, + window: window, + } +} + +func (s *SumRateGenerator) Id() string { + return "" +} + +func (s *SumRateGenerator) Expr() string { + return fmt.Sprintf("sum(rate(%s[%s]))", s.goodExpr, s.window.WindowDur.String()) +} + +func (s *SumRateGenerator) Rule() (rulefmt.RuleNode, error) { + return rulefmt.RuleNode{}, fmt.Errorf("partial rules cannot be converted to rulefmt") +} + +func NewMWMBSLIGenerator( + name, goodExpr, totalExpr string, + window WindowMetadata, + idLabels backend.IdentificationLabels, +) *MWMBSLIGenerator { + goodQuery := NewSumRateGenerator(goodExpr, window) + totalQuery := NewSumRateGenerator(totalExpr, window) + return &MWMBSLIGenerator{ + name: name, + window: window, + goodQuery: goodQuery, + totalQuery: totalQuery, + idLabels: idLabels, + } +} + +type MWMBSLIGenerator struct { + name string + window WindowMetadata + + goodQuery MetricGenerator + totalQuery MetricGenerator + + idLabels backend.IdentificationLabels +} + +var _ MetricGenerator = (*MWMBSLIGenerator)(nil) + +func (s *MWMBSLIGenerator) Id() string { + return s.name + ":" + s.window.Name +} + +func (s *MWMBSLIGenerator) Expr() string { + return fmt.Sprintf("1 - ((%s) / (%s))", s.goodQuery.Expr(), s.totalQuery.Expr()) +} + +func (s *MWMBSLIGenerator) Rule() (rulefmt.RuleNode, error) { + return rulefmt.RuleNode{ + Record: yaml.Node{ + Kind: yaml.ScalarNode, + Value: s.Id(), + }, + Expr: yaml.Node{ + Kind: yaml.ScalarNode, + Value: s.Expr(), + }, + Labels: s.idLabels, + }, nil +} + +type BurnRateGenerator struct { + name string + idLabels backend.IdentificationLabels + + sli MetricGenerator + errorBudget MetricGenerator + + options *SLOGeneratorOptions +} + +var _ MetricGenerator = (*BurnRateGenerator)(nil) + +func (b *BurnRateGenerator) Id() string { + return b.name +} + +func NewBurnRateGenerator( + id string, + idLabels backend.IdentificationLabels, + sli MetricGenerator, + errorBudget MetricGenerator, + options *SLOGeneratorOptions, +) *BurnRateGenerator { + return &BurnRateGenerator{ + name: id, + idLabels: idLabels, + sli: sli, + errorBudget: errorBudget, + options: options, + } +} + +func (b *BurnRateGenerator) Expr() string { + var numerator, denominator string + if b.options.optimized { + numerator = QueryWithLabels(b.sli.Id(), b.idLabels) + denominator = QueryWithLabels(b.errorBudget.Id(), b.idLabels) + return fmt.Sprintf( + "%s / on(%s) group_left %s", + numerator, + b.idLabels.JoinOnPrometheus(), + denominator, + ) + } + numerator = b.sli.Expr() + denominator = b.errorBudget.Expr() + return fmt.Sprintf( + "(%s) / (%s)", + numerator, + denominator, + ) +} + +func (b *BurnRateGenerator) Rule() (rulefmt.RuleNode, error) { + return rulefmt.RuleNode{ + Record: yaml.Node{ + Kind: yaml.ScalarNode, + Value: b.name, + }, + Expr: yaml.Node{ + Kind: yaml.ScalarNode, + Value: b.Expr(), + }, + Labels: b.idLabels, + }, nil +} + +type ErrorBudgetRemainingGenerator struct { + name string + idLabels backend.IdentificationLabels + periodBurnRate MetricGenerator + options *SLOGeneratorOptions +} + +var _ MetricGenerator = (*ErrorBudgetRemainingGenerator)(nil) + +func NewErrorBudgetRemainingGenerator( + id string, + idLabels backend.IdentificationLabels, + periodBurnRate MetricGenerator, + options *SLOGeneratorOptions, +) *ErrorBudgetRemainingGenerator { + return &ErrorBudgetRemainingGenerator{ + name: id, + idLabels: idLabels, + periodBurnRate: periodBurnRate, + options: options, + } +} + +func (e *ErrorBudgetRemainingGenerator) Id() string { + return e.name +} + +func (e *ErrorBudgetRemainingGenerator) Expr() string { + var subExpr string + if e.options.optimized { + subExpr = QueryWithLabels(e.periodBurnRate.Id(), e.idLabels) + } else { + subExpr = e.periodBurnRate.Expr() + } + return fmt.Sprintf("1 - (%s)", subExpr) +} + +func (e *ErrorBudgetRemainingGenerator) Rule() (rulefmt.RuleNode, error) { + return rulefmt.RuleNode{ + Record: yaml.Node{ + Kind: yaml.ScalarNode, + Value: e.name, + }, + Expr: yaml.Node{ + Kind: yaml.ScalarNode, + Value: e.Expr(), + }, + Labels: e.idLabels, + }, nil +} + +// TODO : add a description of how these alerts are structured +type MWMBAlertGenerator struct { + quickWindow Window + slowWindow Window + idLabels backend.IdentificationLabels + options *SLOGeneratorOptions + errorRate func(WindowMetadata) MetricGenerator + asRule bool +} + +var _ MetricGenerator = (*MWMBAlertGenerator)(nil) + +func NewMWMBAlertGenerator( + idLabels backend.IdentificationLabels, + quickWindow, slowWindow Window, + errorRate func(WindowMetadata) MetricGenerator, + options *SLOGeneratorOptions, + asRule bool, +) *MWMBAlertGenerator { + return &MWMBAlertGenerator{ + idLabels: idLabels, + quickWindow: quickWindow, + slowWindow: slowWindow, + errorRate: errorRate, + options: options, + asRule: asRule, + } +} + +func (m *MWMBAlertGenerator) Id() string { + baseID := "slo:mwmb_alert" + if m.asRule { + return baseID + ":interval:" + m.quickWindow.Name() + } + return baseID + ":" + m.quickWindow.Name() +} + +func (m *MWMBAlertGenerator) Expr() string { + construct := func(w WindowMetadata) string { + return m.errorRate(w).Expr() + } + if m.options.optimized { + construct = func(w WindowMetadata) string { + return fmt.Sprintf("%s{%s}", m.errorRate(w).Id(), m.idLabels.ToLabels().ConstructPrometheus()) + } + + } + compOperator := ">" + if m.asRule { + compOperator = "> bool" + } + shortWindowQuickExpr := fmt.Sprintf( + "(%s) %s (%.9f * %.9f)", + construct(m.quickWindow.ShortWindowMetadata()), + compOperator, + m.quickWindow.GetShortBurnRateFactor(), + m.quickWindow.ErrorBudgetPercent, + ) + shortWindowLongExpr := fmt.Sprintf( + "(%s) %s (%.9f * %.9f)", + construct(m.quickWindow.LongWindowMetadata()), + compOperator, + m.quickWindow.GetLongBurnRateFactor(), + m.quickWindow.ErrorBudgetPercent, + ) + longWindowQuickExpr := fmt.Sprintf( + "(%s) %s (%.9f * %.9f)", + construct(m.slowWindow.ShortWindowMetadata()), + compOperator, + m.slowWindow.GetShortBurnRateFactor(), + m.slowWindow.ErrorBudgetPercent, + ) + longWindowLongExpr := fmt.Sprintf( + "(%s) %s (%.9f * %.9f)", + construct(m.slowWindow.LongWindowMetadata()), + compOperator, + m.slowWindow.GetLongBurnRateFactor(), + m.slowWindow.ErrorBudgetPercent, + ) + + return fmt.Sprintf( + "(max((%s)) and max((%s))) or (max(%s) and max(%s))", + shortWindowQuickExpr, + shortWindowLongExpr, + longWindowQuickExpr, + longWindowLongExpr, + ) +} + +func (m *MWMBAlertGenerator) Rule() (rulefmt.RuleNode, error) { + return rulefmt.RuleNode{ + Alert: yaml.Node{ + Kind: yaml.ScalarNode, + Value: m.Id(), + }, + Expr: yaml.Node{ + Kind: yaml.ScalarNode, + Value: m.Expr(), + }, + Labels: m.idLabels, + }, nil +} diff --git a/pkg/slo/backend/metrics/prometheus_test.go b/pkg/slo/backend/metrics/prometheus_test.go new file mode 100644 index 0000000000..3520b10843 --- /dev/null +++ b/pkg/slo/backend/metrics/prometheus_test.go @@ -0,0 +1,993 @@ +package metrics_test + +import ( + "fmt" + "time" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + prommodel "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/rulefmt" + promql "github.com/prometheus/prometheus/promql/parser" + "github.com/rancher/opni/pkg/slo/backend" + "github.com/rancher/opni/pkg/slo/backend/metrics" + "github.com/rancher/opni/pkg/util" + "github.com/samber/lo" +) + +var _ = Describe("Prometheus SLO", Label("unit"), func() { + When("We use metric generators", func() { + Specify("constant generators should produce valid promQL", func() { + cnst := metrics.NewConstantMetricGenerator("hello_vector", 0, 9, map[string]string{ + "foo ": "bar", + }) + + Expect(cnst.Id()).To(Equal("hello_vector")) + Expect(cnst.Expr()).To(Equal("vector(0.000000000)")) + + _, err := promql.ParseExpr(cnst.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err := cnst.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(cnst.Id())) + Expect(rule.Expr.Value).To(Equal(cnst.Expr())) + Expect(rule.Labels).To(Equal(map[string]string{ + "foo ": "bar", + })) + }) + + Specify("sum rate generators should produce valid promQL", func() { + sr := metrics.NewSumRateGenerator("hello_vector", metrics.WindowMetadata{WindowDur: prommodel.Duration(time.Minute)}) + + // this is a partial generator + Expect(sr.Id()).To(Equal("")) + Expect(sr.Expr()).To(Equal("sum(rate(hello_vector[1m]))")) + _, err := sr.Rule() + Expect(err).To(HaveOccurred()) + + _, err = promql.ParseExpr(sr.Expr()) + Expect(err).NotTo(HaveOccurred()) + + seconds := metrics.NewSumRateGenerator("hello_vector", metrics.WindowMetadata{WindowDur: prommodel.Duration(time.Second)}) + Expect(seconds.Id()).To(Equal("")) + Expect(seconds.Expr()).To(Equal("sum(rate(hello_vector[1s]))")) + + _, err = seconds.Rule() + Expect(err).To(HaveOccurred()) + + _, err = promql.ParseExpr(seconds.Expr()) + Expect(err).NotTo(HaveOccurred()) + + hours := metrics.NewSumRateGenerator("hello_vector", metrics.WindowMetadata{WindowDur: prommodel.Duration(time.Hour)}) + Expect(hours.Id()).To(Equal("")) + Expect(hours.Expr()).To(Equal("sum(rate(hello_vector[1h]))")) + + _, err = hours.Rule() + Expect(err).To(HaveOccurred()) + + days := metrics.NewSumRateGenerator("hello_vector", metrics.WindowMetadata{WindowDur: prommodel.Duration(24 * time.Hour)}) + Expect(days.Id()).To(Equal("")) + Expect(days.Expr()).To(Equal("sum(rate(hello_vector[1d]))")) + + _, err = days.Rule() + Expect(err).To(HaveOccurred()) + }) + + Specify("MWMB SLI generators should produce valid promQL", func() { + mwmb := metrics.NewMWMBSLIGenerator( + "sli_vector", + "http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}", + "http_request_duration_seconds_count{job=\"myservice\"}", + metrics.WindowMetadata{WindowDur: prommodel.Duration(time.Minute), Name: "page:quick:short"}, + map[string]string{ + "bar": "baz", + }, + ) + + Expect(mwmb.Id()).To(Equal("sli_vector:page:quick:short")) + Expect(mwmb.Expr()).To(Equal( + "1 - ((sum(rate(http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}[1m]))) / (sum(rate(http_request_duration_seconds_count{job=\"myservice\"}[1m]))))", + )) + + _, err := promql.ParseExpr(mwmb.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err := mwmb.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Labels).To(Equal(map[string]string{ + "bar": "baz", + })) + + _, err = promql.ParseExpr(rule.Record.Value) + Expect(err).NotTo(HaveOccurred()) + + seconds := metrics.NewMWMBSLIGenerator( + "sli_vector", + "http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}", + "http_request_duration_seconds_count{job=\"myservice\"}", + metrics.WindowMetadata{WindowDur: prommodel.Duration(time.Second), Name: "ticket:long:long"}, + map[string]string{ + "bar": "baz", + }, + ) + + Expect(seconds.Id()).To(Equal("sli_vector:ticket:long:long")) + Expect(seconds.Expr()).To(Equal( + "1 - ((sum(rate(http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}[1s]))) / (sum(rate(http_request_duration_seconds_count{job=\"myservice\"}[1s]))))", + )) + + _, err = promql.ParseExpr(seconds.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = seconds.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Labels).To(Equal(map[string]string{ + "bar": "baz", + })) + + _, err = promql.ParseExpr(rule.Record.Value) + Expect(err).NotTo(HaveOccurred()) + + hours := metrics.NewMWMBSLIGenerator( + "sli_vector", + "http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}", + "http_request_duration_seconds_count{job=\"myservice\"}", + metrics.WindowMetadata{WindowDur: prommodel.Duration(time.Hour * 6), Name: "page:quick:short"}, + map[string]string{ + "bar": "baz", + }, + ) + + Expect(hours.Id()).To(Equal("sli_vector:page:quick:short")) + Expect(hours.Expr()).To(Equal( + "1 - ((sum(rate(http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}[6h]))) / (sum(rate(http_request_duration_seconds_count{job=\"myservice\"}[6h]))))", + )) + + _, err = promql.ParseExpr(hours.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = hours.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Labels).To(Equal(map[string]string{ + "bar": "baz", + })) + + _, err = promql.ParseExpr(rule.Record.Value) + Expect(err).NotTo(HaveOccurred()) + + days := metrics.NewMWMBSLIGenerator( + "sli_vector", + "http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}", + "http_request_duration_seconds_count{job=\"myservice\"}", + metrics.WindowMetadata{WindowDur: prommodel.Duration(time.Hour * 24 * 24), Name: "test"}, + map[string]string{ + "bar": "baz", + }, + ) + + Expect(days.Id()).To(Equal("sli_vector:test")) + Expect(days.Expr()).To(Equal( + "1 - ((sum(rate(http_request_duration_seconds_count{job=\"myservice\", code=~\"5..|429\"}[24d]))) / (sum(rate(http_request_duration_seconds_count{job=\"myservice\"}[24d]))))", + )) + + _, err = promql.ParseExpr(days.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = days.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Labels).To(Equal(map[string]string{ + "bar": "baz", + })) + + _, err = promql.ParseExpr(rule.Record.Value) + Expect(err).NotTo(HaveOccurred()) + }) + + Specify("Burn rate generators should generate valid promQL", func() { + idLabels := map[string]string{ + "id": "1", + } + + valueGenerator1 := metrics.NewConstantMetricGenerator( + "test_metric", + 0.5, + 9, + idLabels, + ) + + valueGenerator2 := metrics.NewConstantMetricGenerator( + "test_metric2", + 0.4, + 9, + idLabels, + ) + + opts := metrics.DefaultSLOGeneratorOptions() + + br := metrics.NewBurnRateGenerator( + "burn_rate_current", + idLabels, + valueGenerator1, + valueGenerator2, + opts, + ) + + Expect(br.Id()).To(Equal("burn_rate_current")) + Expect(br.Expr()).To(Equal("test_metric{id=~\"1\"} / on(id) group_left test_metric2{id=~\"1\"}")) + + _, err := promql.ParseExpr(br.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err := br.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Labels).To(Equal(idLabels)) + Expect(rule.Record.Value).To(Equal(br.Id())) + Expect(rule.Expr.Value).To(Equal(br.Expr())) + + idLabels2 := map[string]string{ + "foo": "bar", + "baz": "qux", + } + + br2 := metrics.NewBurnRateGenerator( + "burn_rate_current", + idLabels2, + valueGenerator1, + valueGenerator2, + opts, + ) + + Expect(br2.Id()).To(Equal("burn_rate_current")) + Expect(br2.Expr()).To(Equal("test_metric{baz=~\"qux\",foo=~\"bar\"} / on(baz, foo) group_left test_metric2{baz=~\"qux\",foo=~\"bar\"}")) + + _, err = promql.ParseExpr(br2.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = br2.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Labels).To(Equal(idLabels2)) + Expect(rule.Record.Value).To(Equal(br2.Id())) + Expect(rule.Expr.Value).To(Equal(br2.Expr())) + + By("verifying unpotimized builds return runnable queries") + + opts.Apply( + metrics.WithOptimization(false), + ) + + brUnopt := metrics.NewBurnRateGenerator( + "burn_rate_current", + idLabels2, + valueGenerator1, + valueGenerator2, + opts, + ) + Expect(brUnopt.Id()).To(Equal("burn_rate_current")) + Expect(brUnopt.Expr()).To(Equal("(vector(0.500000000)) / (vector(0.400000000))")) + + _, err = promql.ParseExpr(brUnopt.Expr()) + Expect(err).NotTo(HaveOccurred()) + }) + + Specify("Error budget generators should generate valid promQL", func() { + opts := metrics.DefaultSLOGeneratorOptions() + idLabels := map[string]string{ + "id": "1", + "foo": "bar", + } + periodQuery := metrics.NewConstantMetricGenerator( + "test_metric", + 0.5, + 9, + idLabels, + ) + + errB := metrics.NewErrorBudgetRemainingGenerator( + "hello", + idLabels, + periodQuery, + opts, + ) + Expect(errB.Id()).To(Equal("hello")) + Expect(errB.Expr()).To(Equal("1 - (test_metric{foo=~\"bar\",id=~\"1\"})")) + + _, err := promql.ParseExpr(errB.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err := errB.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Labels).To(Equal(idLabels)) + Expect(rule.Record.Value).To(Equal(errB.Id())) + Expect(rule.Expr.Value).To(Equal(errB.Expr())) + + By("verifying unpotimized builds return runnable queries") + + opts.Apply( + metrics.WithOptimization(false), + ) + + errBUnopt := metrics.NewErrorBudgetRemainingGenerator( + "hello", + idLabels, + periodQuery, + opts, + ) + Expect(errBUnopt.Id()).To(Equal("hello")) + Expect(errBUnopt.Expr()).To(Equal(fmt.Sprintf("1 - (%s)", periodQuery.Expr()))) + + _, err = promql.ParseExpr(errBUnopt.Expr()) + Expect(err).NotTo(HaveOccurred()) + }) + + Specify("MWMB alerts should generate valid promQL", func() { + opts := metrics.DefaultSLOGeneratorOptions() + idLabels := map[string]string{ + "id": "1", + "foo": "bar", + } + errorRate := func(window metrics.WindowMetadata) metrics.MetricGenerator { + return metrics.NewMWMBSLIGenerator( + "error_rate", + "test_metric{foo=~\"bar\",id=~\"1\"}", + "test_metric{foo=~\"bar\",id=~\"1|2\"}", + window, + idLabels, + ) + } + + mwmb := metrics.WindowDefaults(prommodel.Duration(time.Hour * 24 * 30)) + quick, long := mwmb.PageWindows() + pageGen := metrics.NewMWMBAlertGenerator( + idLabels, + quick, + long, + errorRate, + opts, + false, + ) + + Expect(pageGen.Id()).To(Equal("slo:mwmb_alert:page:quick")) + GinkgoWriter.Write([]byte(pageGen.Expr())) + expected := "(max(((error_rate:page:quick:short{foo=~\"bar\",id=~\"1\"}) > (172.800000000 * 2.000000000))) and max(((error_rate:page:quick:long{foo=~\"bar\",id=~\"1\"}) > (14.400000000 * 2.000000000)))) or (max((error_rate:page:slow:short{foo=~\"bar\",id=~\"1\"}) > (72.000000000 * 5.000000000)) and max((error_rate:page:slow:long{foo=~\"bar\",id=~\"1\"}) > (6.000000000 * 5.000000000)))" + Expect(pageGen.Expr()).Should(Equal(expected)) + + rule, err := pageGen.Rule() + Expect(err).NotTo(HaveOccurred()) + + _, err = promql.ParseExpr(rule.Expr.Value) + Expect(err).NotTo(HaveOccurred()) + + Expect(rule.Alert.Value).To(Equal(pageGen.Id())) + Expect(rule.Expr.Value).To(Equal(pageGen.Expr())) + Expect(rule.Labels).To(Equal(idLabels)) + + By("verifying unpotimized MWMB alerts generate valid promQL") + + opts.Apply( + metrics.WithOptimization(false), + ) + pageGenRaw := metrics.NewMWMBAlertGenerator( + idLabels, + quick, + long, + errorRate, + opts, + false, + ) + + Expect(pageGenRaw.Id()).To(Equal("slo:mwmb_alert:page:quick")) + expectedRaw := "(max(((1 - ((sum(rate(test_metric{foo=~\"bar\",id=~\"1\"}[5m]))) / (sum(rate(test_metric{foo=~\"bar\",id=~\"1|2\"}[5m]))))) > (172.800000000 * 2.000000000))) and max(((1 - ((sum(rate(test_metric{foo=~\"bar\",id=~\"1\"}[1h]))) / (sum(rate(test_metric{foo=~\"bar\",id=~\"1|2\"}[1h]))))) > (14.400000000 * 2.000000000)))) or (max((1 - ((sum(rate(test_metric{foo=~\"bar\",id=~\"1\"}[30m]))) / (sum(rate(test_metric{foo=~\"bar\",id=~\"1|2\"}[30m]))))) > (72.000000000 * 5.000000000)) and max((1 - ((sum(rate(test_metric{foo=~\"bar\",id=~\"1\"}[6h]))) / (sum(rate(test_metric{foo=~\"bar\",id=~\"1|2\"}[6h]))))) > (6.000000000 * 5.000000000)))" + Expect(pageGenRaw.Expr()).Should(Equal(expectedRaw)) + + rule, err = pageGenRaw.Rule() + Expect(err).NotTo(HaveOccurred()) + + _, err = promql.ParseExpr(rule.Expr.Value) + Expect(err).NotTo(HaveOccurred()) + + Expect(rule.Alert.Value).To(Equal(pageGenRaw.Id())) + Expect(rule.Expr.Value).To(Equal(pageGenRaw.Expr())) + Expect(rule.Labels).To(Equal(idLabels)) + + }) + }) + + When("we use the Prometheus SLO generator", func() { + It("should generate valid MWMB optimized recording rules", func() { + periods := []string{"7m", "7d", "30d"} + for _, period := range periods { + sloUuid := uuid.New().String() + idLabels := backend.IdentificationLabels(map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }) + curSloFilters := idLabels.ToLabels().ConstructPrometheus() + Expect(curSloFilters).To(Equal(fmt.Sprintf("slo_opni_id=~\"%s\",slo_opni_name=~\"test\",slo_opni_service=~\"scrape\"", sloUuid))) + curJoinFilters := idLabels.JoinOnPrometheus() + Expect(curJoinFilters).To(Equal("slo_opni_id, slo_opni_name, slo_opni_service")) + + sloGen, err := metrics.NewSLOGenerator( + backend.SLO{ + SloPeriod: period, + Objective: 99.9, + Svc: "scrape", + GoodMetric: "test_metric", + TotalMetric: "test_metric", + IdLabels: idLabels, + UserLabels: map[string]string{}, + GoodEvents: []backend.LabelPair{ + { + Key: "foo", + Vals: []string{"bar"}, + }, + }, + TotalEvents: []backend.LabelPair{}, + }, + ) + Expect(err).NotTo(HaveOccurred()) + windows := sloGen.Windows() + Expect(windows).NotTo(HaveLen(0)) + Expect(len(windows)).To(BeNumerically(">", 2)) + + for _, window := range windows { + + gen := sloGen.SLI(window) + Expect(gen).NotTo(BeNil()) + Expect(gen.Id()).NotTo(Equal("")) + Expect(gen.Id()).To(HavePrefix(metrics.SLOSLI)) + Expect(gen.Id()).To(HaveSuffix(window.Name)) + Expect(gen.Id()).To(Equal(metrics.SLOSLI + ":" + window.Name)) + expected := fmt.Sprintf("1 - ((sum(rate(test_metric{job=\"scrape\", foo=~\"bar\"}[%s]))) / (sum(rate(test_metric{job=\"scrape\"}[%s]))))", prommodel.Duration(window.WindowDur), prommodel.Duration(window.WindowDur)) + Expect(gen.Expr()).To(Equal(expected)) + _, err = promql.ParseExpr(gen.Expr()) + Expect(err).NotTo(HaveOccurred()) + rule, err := gen.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(gen.Id())) + Expect(rule.Expr.Value).To(Equal(gen.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + } + + By("expecting the constant value rules are correct") + infoVec := sloGen.Info() + Expect(infoVec.Id()).To(Equal(metrics.SLOInfo)) + Expect(infoVec.Expr()).To(Equal("vector(1.0)")) + + _, err = promql.ParseExpr(infoVec.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err := infoVec.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(infoVec.Id())) + Expect(rule.Expr.Value).To(Equal(infoVec.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + periodVec := sloGen.Period() + periodFloat := time.Duration(util.Must(prommodel.ParseDuration(period))).Seconds() + Expect(periodVec.Id()).To(Equal(metrics.SLOPeriod)) + Expect(periodVec.Expr()).To(Equal(fmt.Sprintf("vector(%.9f)", periodFloat))) + + _, err = promql.ParseExpr(periodVec.Expr()) + Expect(err).NotTo(HaveOccurred()) + rule, err = periodVec.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(periodVec.Id())) + Expect(rule.Expr.Value).To(Equal(periodVec.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + objective := sloGen.Objective() + Expect(objective.Id()).To(Equal(metrics.SLOObjective)) + Expect(objective.Expr()).To(Equal("vector(0.999000000)")) + + _, err = promql.ParseExpr(objective.Expr()) + Expect(err).NotTo(HaveOccurred()) + rule, err = objective.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(objective.Id())) + + errorBudget := sloGen.ErrorBudget() + Expect(errorBudget.Id()).To(Equal(metrics.SLOErrorBudget)) + Expect(errorBudget.Expr()).To(Equal("vector(0.001000000)")) + + _, err = promql.ParseExpr(errorBudget.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = errorBudget.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(errorBudget.Id())) + Expect(rule.Expr.Value).To(Equal(errorBudget.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + errorBudgetId := sloGen.ErrorBudget().Id() + By("expecting the optmimized current burn rate rule to reference the correct sli rule") + cbr := sloGen.CurrentBurnRate() + Expect(cbr.Id()).To(Equal(metrics.SLOCurrentBurnRate)) + quickestSLIId := sloGen.SLI(sloGen.Windows()[0]).Id() + expected := fmt.Sprintf( + `%s{%s} / on(%s) group_left %s{%s}`, + quickestSLIId, + curSloFilters, + curJoinFilters, + errorBudgetId, + curSloFilters, + ) + Expect(cbr.Expr()).To(Equal(expected)) + + _, err = promql.ParseExpr(cbr.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = cbr.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(cbr.Id())) + Expect(rule.Expr.Value).To(Equal(cbr.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + By("expecting the optimized period burn rate rule to reference the correct sli rule") + pbr := sloGen.PeriodBurnRate() + + periodSLIID := sloGen.SLI(metrics.WindowMetadata{WindowDur: util.Must(prommodel.ParseDuration(period)), Name: "period"}).Id() + + expectedP := fmt.Sprintf( + `%s{%s} / on(%s) group_left %s{%s}`, + periodSLIID, + curSloFilters, + curJoinFilters, + errorBudgetId, + curSloFilters, + ) + GinkgoWriter.Write([]byte(expectedP)) + Expect(pbr.Expr()).To(Equal(expectedP)) + + _, err = promql.ParseExpr(pbr.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = pbr.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(pbr.Id())) + Expect(rule.Expr.Value).To(Equal(pbr.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + By("expecting the optimized error budget remaining rules references the correct metrics") + + errBr := sloGen.ErrorBudgetRemaining() + Expect(errBr.Id()).To(Equal(metrics.SLOErrorBudgetRemaining)) + Expect(errBr.Expr()).To(Equal(fmt.Sprintf("1 - (%s{%s})", pbr.Id(), curSloFilters))) + _, err = promql.ParseExpr(errBr.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = errBr.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(errBr.Id())) + Expect(rule.Expr.Value).To(Equal(errBr.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + } + }) + + It("should generate valid unoptimized MWMB recording rules", func() { + dur := prommodel.Duration((time.Minute*14 + (time.Second * 24)) * 10) + periods := []string{dur.String(), "7d", "30d"} + for _, period := range periods { + sloUuid := uuid.New().String() + idLabels := backend.IdentificationLabels(map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }) + curSloFilters := idLabels.ToLabels().ConstructPrometheus() + Expect(curSloFilters).To(Equal(fmt.Sprintf("slo_opni_id=~\"%s\",slo_opni_name=~\"test\",slo_opni_service=~\"scrape\"", sloUuid))) + curJoinFilters := idLabels.JoinOnPrometheus() + Expect(curJoinFilters).To(Equal("slo_opni_id, slo_opni_name, slo_opni_service")) + + sloGen, err := metrics.NewSLOGenerator( + backend.SLO{ + SloPeriod: period, + Objective: 99.9, + Svc: "scrape", + GoodMetric: "test_metric", + TotalMetric: "test_metric", + IdLabels: idLabels, + UserLabels: map[string]string{}, + GoodEvents: []backend.LabelPair{ + { + Key: "foo", + Vals: []string{"bar"}, + }, + { + Key: "code", + Vals: []string{"200"}, + }, + }, + TotalEvents: []backend.LabelPair{ + { + Key: "code", + Vals: []string{"200", "500", "503"}, + }, + }, + }, + metrics.WithOptimization(false), + ) + Expect(err).NotTo(HaveOccurred()) + windows := sloGen.Windows() + Expect(windows).NotTo(HaveLen(0)) + Expect(len(windows)).To(BeNumerically(">", 2)) + + for _, window := range windows { + gen := sloGen.SLI(window) + Expect(gen).NotTo(BeNil()) + Expect(gen.Id()).NotTo(Equal("")) + Expect(gen.Id()).To(HavePrefix(metrics.SLOSLI)) + Expect(gen.Id()).To(HaveSuffix(window.Name)) + Expect(gen.Id()).To(Equal(metrics.SLOSLI + ":" + window.Name)) + expected := fmt.Sprintf("1 - ((sum(rate(test_metric{job=\"scrape\", foo=~\"bar\",code=~\"200\"}[%s]))) / (sum(rate(test_metric{job=\"scrape\", code=~\"200|500|503\"}[%s]))))", prommodel.Duration(window.WindowDur), prommodel.Duration(window.WindowDur)) + Expect(gen.Expr()).To(Equal(expected)) + _, err := promql.ParseExpr(gen.Expr()) + Expect(err).NotTo(HaveOccurred()) + rule, err := gen.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(gen.Id())) + Expect(rule.Expr.Value).To(Equal(gen.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + } + + By("expecting the constant value rules are correct") + infoVec := sloGen.Info() + Expect(infoVec.Id()).To(Equal(metrics.SLOInfo)) + Expect(infoVec.Expr()).To(Equal("vector(1.0)")) + + _, err = promql.ParseExpr(infoVec.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err := infoVec.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(infoVec.Id())) + Expect(rule.Expr.Value).To(Equal(infoVec.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + periodVec := sloGen.Period() + periodFloat := time.Duration(util.Must(prommodel.ParseDuration(period))).Seconds() + Expect(periodVec.Id()).To(Equal(metrics.SLOPeriod)) + Expect(periodVec.Expr()).To(Equal(fmt.Sprintf("vector(%.9f)", periodFloat))) + + _, err = promql.ParseExpr(periodVec.Expr()) + Expect(err).NotTo(HaveOccurred()) + rule, err = periodVec.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(periodVec.Id())) + Expect(rule.Expr.Value).To(Equal(periodVec.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + objective := sloGen.Objective() + Expect(objective.Id()).To(Equal(metrics.SLOObjective)) + Expect(objective.Expr()).To(Equal("vector(0.999000000)")) + + _, err = promql.ParseExpr(objective.Expr()) + Expect(err).NotTo(HaveOccurred()) + rule, err = objective.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(objective.Id())) + + errorBudget := sloGen.ErrorBudget() + Expect(errorBudget.Id()).To(Equal(metrics.SLOErrorBudget)) + Expect(errorBudget.Expr()).To(Equal("vector(0.001000000)")) + + _, err = promql.ParseExpr(errorBudget.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = errorBudget.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(errorBudget.Id())) + Expect(rule.Expr.Value).To(Equal(errorBudget.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + errorBudgetExpr := sloGen.ErrorBudget().Expr() + By("expecting the unoptmimized current burn rate rule to be valid promQL") + cbr := sloGen.CurrentBurnRate() + Expect(cbr.Id()).To(Equal(metrics.SLOCurrentBurnRate)) + quickestSLIExpr := sloGen.SLI(sloGen.Windows()[0]).Expr() + expected := fmt.Sprintf("(%s) / (%s)", quickestSLIExpr, errorBudgetExpr) + Expect(cbr.Expr()).To(Equal(expected)) + + _, err = promql.ParseExpr(cbr.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = cbr.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(cbr.Id())) + Expect(rule.Expr.Value).To(Equal(cbr.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + By("expecting the unoptimized period burn rate rule to be valid promql") + pbr := sloGen.PeriodBurnRate() + periodSLIExpr := sloGen.SLI(metrics.WindowMetadata{WindowDur: util.Must(prommodel.ParseDuration(period)), Name: "test"}).Expr() + + expectedP := fmt.Sprintf( + `(%s) / (%s)`, + periodSLIExpr, + errorBudgetExpr, + ) + GinkgoWriter.Write([]byte(expectedP)) + Expect(pbr.Expr()).To(Equal(expectedP)) + + _, err = promql.ParseExpr(pbr.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = pbr.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(pbr.Id())) + Expect(rule.Expr.Value).To(Equal(pbr.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + + By("expecting the unoptimized error budget remaining rules references the correct metrics") + + errBr := sloGen.ErrorBudgetRemaining() + Expect(errBr.Id()).To(Equal(metrics.SLOErrorBudgetRemaining)) + Expect(errBr.Expr()).To(Equal(fmt.Sprintf("1 - (%s)", pbr.Expr()))) + + _, err = promql.ParseExpr(errBr.Expr()) + Expect(err).NotTo(HaveOccurred()) + + rule, err = errBr.Rule() + Expect(err).NotTo(HaveOccurred()) + Expect(rule.Record.Value).To(Equal(errBr.Id())) + Expect(rule.Expr.Value).To(Equal(errBr.Expr())) + Expect(rule.Labels).To(Equal( + map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + }, + )) + } + }) + It("should generate valid optimized rule groups", func() { + sloUuid := uuid.New().String() + idLabels := map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + } + curSloFilters := backend.IdentificationLabels(idLabels).ToLabels().ConstructPrometheus() + Expect(curSloFilters).To(Equal(fmt.Sprintf("slo_opni_id=~\"%s\",slo_opni_name=~\"test\",slo_opni_service=~\"scrape\"", sloUuid))) + curJoinFilters := backend.IdentificationLabels(idLabels).JoinOnPrometheus() + Expect(curJoinFilters).To(Equal("slo_opni_id, slo_opni_name, slo_opni_service")) + + period := prommodel.Duration((time.Minute*14 + (time.Second * 24)) * 10) + + sloGen, err := metrics.NewSLOGenerator( + backend.SLO{ + SloPeriod: period.String(), + Objective: 99.9, + Svc: "scrape", + GoodMetric: "test_metric", + TotalMetric: "test_metric", + IdLabels: idLabels, + UserLabels: map[string]string{}, + GoodEvents: []backend.LabelPair{ + { + Key: "foo", + Vals: []string{"bar"}, + }, + { + Key: "code", + Vals: []string{"200"}, + }, + }, + TotalEvents: []backend.LabelPair{ + { + Key: "code", + Vals: []string{"200", "500", "503"}, + }, + }, + }, + metrics.WithOptimization(false), + ) + Expect(err).NotTo(HaveOccurred()) + + rules, err := sloGen.AsRuleGroup() + Expect(err).NotTo(HaveOccurred()) + Expect(rules).NotTo(BeNil()) + Expect(rules.Name).To(ContainSubstring(sloUuid)) + Expect(rules.Rules).NotTo(HaveLen(0)) + + By("verifying the ouput rule groups contain the correct rules") + ruleIds := lo.Map(rules.Rules, func(r rulefmt.RuleNode, _ int) string { + if r.Record.Value == "" { + return r.Alert.Value + } + return r.Record.Value + }) + + generatorIds := []string{ + sloGen.Info().Id(), + sloGen.Objective().Id(), + sloGen.ErrorBudget().Id(), + sloGen.Period().Id(), + sloGen.CurrentBurnRate().Id(), + sloGen.PeriodBurnRate().Id(), + sloGen.ErrorBudgetRemaining().Id(), + } + for _, window := range append(sloGen.Windows()) { + generatorIds = append(generatorIds, sloGen.SLI(window).Id()) + } + + Expect(ruleIds).To(ConsistOf(generatorIds)) + + By("verifying the contents are valid promQL and are labelled with the correct ID") + + for _, rule := range rules.Rules { + _, err := promql.ParseExpr(rule.Expr.Value) + Expect(err).NotTo(HaveOccurred()) + + Expect(rule.Labels).To(Equal( + idLabels, + )) + } + }) + + It("should generated valid unoptimized rule groups", func() { + sloUuid := uuid.New().String() + idLabels := map[string]string{ + backend.SLOUuid: sloUuid, + backend.SLOName: "test", + backend.SLOService: "scrape", + } + curSloFilters := backend.IdentificationLabels(idLabels).ToLabels().ConstructPrometheus() + Expect(curSloFilters).To(Equal(fmt.Sprintf("slo_opni_id=~\"%s\",slo_opni_name=~\"test\",slo_opni_service=~\"scrape\"", sloUuid))) + curJoinFilters := backend.IdentificationLabels(idLabels).JoinOnPrometheus() + Expect(curJoinFilters).To(Equal("slo_opni_id, slo_opni_name, slo_opni_service")) + + period := prommodel.Duration((time.Minute*14 + (time.Second * 24)) * 10) + + sloGen, err := metrics.NewSLOGenerator( + backend.SLO{ + SloPeriod: period.String(), + Objective: 99.9, + Svc: "scrape", + GoodMetric: "test_metric", + TotalMetric: "test_metric", + IdLabels: idLabels, + UserLabels: map[string]string{}, + GoodEvents: []backend.LabelPair{ + { + Key: "foo", + Vals: []string{"bar"}, + }, + { + Key: "code", + Vals: []string{"200"}, + }, + }, + TotalEvents: []backend.LabelPair{ + { + Key: "code", + Vals: []string{"200", "500", "503"}, + }, + }, + }, + metrics.WithOptimization(false), + ) + Expect(err).NotTo(HaveOccurred()) + + rules, err := sloGen.AsRuleGroup() + Expect(err).NotTo(HaveOccurred()) + Expect(rules).NotTo(BeNil()) + Expect(rules.Name).To(ContainSubstring(sloUuid)) + Expect(rules.Rules).NotTo(HaveLen(0)) + + By("verifying the ouput rule groups contain the correct rules") + ruleIds := lo.Map(rules.Rules, func(r rulefmt.RuleNode, _ int) string { + if r.Record.Value == "" { + return r.Alert.Value + } + return r.Record.Value + }) + + generatorIds := []string{ + sloGen.Info().Id(), + sloGen.Objective().Id(), + sloGen.ErrorBudget().Id(), + sloGen.Period().Id(), + sloGen.CurrentBurnRate().Id(), + sloGen.PeriodBurnRate().Id(), + sloGen.ErrorBudgetRemaining().Id(), + } + for _, window := range append(sloGen.Windows()) { + generatorIds = append(generatorIds, sloGen.SLI(window).Id()) + } + + Expect(ruleIds).To(ConsistOf(generatorIds)) + + By("verifying the contents are valid promQL and are labelled with the correct ID") + + for _, rule := range rules.Rules { + _, err := promql.ParseExpr(rule.Expr.Value) + Expect(err).NotTo(HaveOccurred()) + + Expect(rule.Labels).To(Equal( + idLabels, + )) + } + }) + }) +}) diff --git a/plugins/slo/pkg/slo/servicegroups/kubernetes.yaml b/pkg/slo/backend/metrics/servicegroups/kubernetes.yaml similarity index 87% rename from plugins/slo/pkg/slo/servicegroups/kubernetes.yaml rename to pkg/slo/backend/metrics/servicegroups/kubernetes.yaml index c7960647a6..23775b3884 100644 --- a/plugins/slo/pkg/slo/servicegroups/kubernetes.yaml +++ b/pkg/slo/backend/metrics/servicegroups/kubernetes.yaml @@ -2,20 +2,18 @@ name: "kubernetes metrics" filters: - value : ".*apiserver.*" score : 8 - - value : ".*kube-state-metrics.*" - score : 8 - - value : ".*node-exporter.*" - score : 8 - - value : ".*kube.*" - score : 4 -ignore: - value : ".*kube-proxy.*" score : 10 + - value : ".*kube.*" + score : 4 - value : ".*kubelet.*" score : 8 - - value : ".*coredns.*" +ignore: + - value : ".*kube-state-metrics.*" score : 8 - - value : ".*kube-promethe-operator.*" + - value : ".*node-exporter.*" + score : 8 + - value : ".*coredns.*" score : 8 - value : ".*kube-prometheus-operator.*" score : 8 \ No newline at end of file diff --git a/pkg/slo/backend/metrics/window.go b/pkg/slo/backend/metrics/window.go new file mode 100644 index 0000000000..61f910e275 --- /dev/null +++ b/pkg/slo/backend/metrics/window.go @@ -0,0 +1,269 @@ +package metrics + +import ( + "fmt" + "time" + + "slices" + + "github.com/prometheus/common/model" +) + +func NormalizePeriodToBudgetingInterval(period time.Duration) (budgetingInterval time.Duration) { + return time.Duration( + int((float64(time.Duration(period).Nanoseconds()) * (float64(5) / float64(43200)))), + ) +} + +type MWMBWindows struct { + Period model.Duration + PageQuick Window + PageSlow Window + TicketQuick Window + TicketSlow Window +} + +func (w MWMBWindows) windows() []Window { + return []Window{ + w.PageQuick, + w.PageSlow, + w.TicketQuick, + w.TicketSlow, + } +} + +type WindowMetadata struct { + WindowDur model.Duration + // use for building other rules + Name string +} + +func (w MWMBWindows) WindowRange() []WindowMetadata { + windows := []WindowMetadata{} + for _, w := range w.windows() { + windows = append(windows, w.ShortWindowMetadata()) + windows = append(windows, w.LongWindowMetadata()) + } + windows = append(windows, WindowMetadata{ + WindowDur: w.Period, + Name: "period", + }) + + slices.SortFunc(windows, func(i, j WindowMetadata) int { + return int(int64(i.WindowDur) - int64(j.WindowDur)) + }) + return windows +} + +// https://sre.google/workbook/alerting-on-slos/ +// +// budgeting interval is the shortest interval to monitor in a window +// +// budgeting interval should roughly be set to 5/432 of the SLO period +func GenerateMWMBWindows(sloPeriod, budgetingInterval model.Duration) *MWMBWindows { + pageQuickShort := budgetingInterval + pageQuickLong := budgetingInterval * 12 + + pageSlowShort := budgetingInterval * 6 + pageSlowLong := pageSlowShort * 12 + + ticketQuickShort := pageSlowShort * 4 + ticketQuickLong := ticketQuickShort * 12 + + ticketSlowShort := ticketQuickShort * 3 + ticketSlowLong := ticketQuickLong * 3 + + return &MWMBWindows{ + Period: sloPeriod, + PageQuick: Window{ + LongWindow: pageQuickLong, + ShortWindow: pageQuickShort, + ErrorBudgetPercent: 2, + Period: sloPeriod, + name: "page:quick", + }, + PageSlow: Window{ + LongWindow: pageSlowLong, + ShortWindow: pageSlowShort, + ErrorBudgetPercent: 5, + Period: sloPeriod, + name: "page:slow", + }, + TicketQuick: Window{ + LongWindow: ticketQuickLong, + ShortWindow: ticketQuickShort, + ErrorBudgetPercent: 10, + Period: sloPeriod, + name: "ticket:quick", + }, + TicketSlow: Window{ + LongWindow: ticketSlowLong, + ShortWindow: ticketSlowShort, + ErrorBudgetPercent: 10, + Period: sloPeriod, + name: "ticket:slow", + }, + } +} + +// https://sre.google/workbook/alerting-on-slos/ +func WindowDefaults(period model.Duration) *MWMBWindows { + return &MWMBWindows{ + Period: period, + PageQuick: Window{ + ShortWindow: model.Duration(time.Minute) * 5, + LongWindow: model.Duration(time.Minute) * 60, + ErrorBudgetPercent: 2, + Period: period, + name: "page:quick", + }, + PageSlow: Window{ + ShortWindow: model.Duration(time.Minute) * 30, + LongWindow: model.Duration(time.Hour) * 6, + ErrorBudgetPercent: 5, + Period: period, + name: "page:slow", + }, + TicketQuick: Window{ + LongWindow: (model.Duration(time.Hour) * 24), + ShortWindow: model.Duration(time.Hour) * 2, + ErrorBudgetPercent: 10, + Period: period, + name: "ticket:quick", + }, + TicketSlow: Window{ + LongWindow: (model.Duration(time.Hour) * 24) * 3, + ShortWindow: model.Duration(time.Hour) * 6, + ErrorBudgetPercent: 10, + Period: period, + name: "ticket:slow", + }, + } +} + +type Window struct { + // ErrorBudgetPercent is the error budget % consumed for a full time window. + // Google gives us some defaults in its SRE workbook that work correctly most of the times: + // - Page quick: 2% + // - Page slow: 5% + // - Ticket quick: 10% + // - Ticket slow: 10% + ErrorBudgetPercent float64 + // ShortWindow is the small window used on the alerting part to stop alerting + // during a long window because we consumed a lot of error budget but the problem + // is already gone. + ShortWindow model.Duration + // LongWindow is the long window used to alert based on the errors happened on that + // long window. + LongWindow model.Duration + // Overall SLO period + Period model.Duration + // Name of the current window, builds into prometheus rule names that depend on this window. + name string +} + +func (w Window) ShortWindowMetadata() WindowMetadata { + return WindowMetadata{ + WindowDur: w.ShortWindow, + Name: w.Name() + ":short", + } +} + +func (w Window) LongWindowMetadata() WindowMetadata { + return WindowMetadata{ + WindowDur: w.LongWindow, + Name: w.Name() + ":long", + } +} + +func (w Window) Name() string { + return w.name +} + +func (w Window) Validate() error { + if w.Period == 0 { + return fmt.Errorf("period is required") + } + if w.LongWindow == 0 { + return fmt.Errorf("long window is required") + } + + if w.ShortWindow == 0 { + return fmt.Errorf("short window is required") + } + + if w.ErrorBudgetPercent == 0 { + return fmt.Errorf("error budget is required") + } + + return nil +} + +func (w MWMBWindows) Validate() error { + + err := w.PageQuick.Validate() + if err != nil { + return fmt.Errorf("invalid page quick: %w", err) + } + + err = w.PageSlow.Validate() + if err != nil { + return fmt.Errorf("invalid page slow: %w", err) + } + + err = w.TicketQuick.Validate() + if err != nil { + return fmt.Errorf("invalid ticket quick: %w", err) + } + + err = w.TicketSlow.Validate() + if err != nil { + return fmt.Errorf("invalid ticket slow: %w", err) + } + + if w.Period < w.TicketSlow.LongWindow || + w.Period < w.TicketSlow.ShortWindow || + w.Period < w.TicketQuick.LongWindow || + w.Period < w.TicketQuick.ShortWindow { + return fmt.Errorf("period must be greater than ticket budgeting intervals") + } + + if w.Period < w.PageSlow.LongWindow || + w.Period < w.PageSlow.ShortWindow || + w.Period < w.PageQuick.LongWindow || + w.Period < w.PageQuick.ShortWindow { + return fmt.Errorf("period must be greater than page budgeting intervals") + } + + return nil +} + +func (w MWMBWindows) PageWindows() (quickWindow, slowWindow Window) { + return w.PageQuick, w.PageSlow +} + +func (w MWMBWindows) TicketWindows() (quickWindow, slowWindow Window) { + return w.TicketQuick, w.TicketSlow +} + +// Error budget speeds based on a full time window, however once we have the factor (speed) +// the value can be used with any time window. +func (w Window) GetLongBurnRateFactor() float64 { + return GetBurnRateFactor(w.Period, w.ErrorBudgetPercent, w.LongWindow) +} + +func (w Window) GetShortBurnRateFactor() float64 { + return GetBurnRateFactor(w.Period, w.ErrorBudgetPercent, w.ShortWindow) +} + +// getBurnRateFactor calculates the burnRateFactor (speed) needed to consume all the error budget available percent +// in a specific time window taking into account the total time window. +// returns a value between 0 and 100 +func GetBurnRateFactor(totalWindow model.Duration, errorBudgetPercent float64, consumptionWindow model.Duration) float64 { + // First get the total seconds required to consume the % of the error budget in the total window. + secondsRequiredConsumption := (errorBudgetPercent * time.Duration(totalWindow).Seconds()) / 100 + // Now calculate how much is the factor required for the hours consumption, in case we would need to use + // a different time window (e.g: hours required: 36h, if we want to do it in 6h: would be `x6`). + speed := secondsRequiredConsumption / time.Duration(consumptionWindow).Seconds() + return speed +} diff --git a/pkg/slo/backend/metrics/window_test.go b/pkg/slo/backend/metrics/window_test.go new file mode 100644 index 0000000000..cc05a7246d --- /dev/null +++ b/pkg/slo/backend/metrics/window_test.go @@ -0,0 +1,72 @@ +package metrics_test + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/common/model" + "github.com/rancher/opni/pkg/slo/backend/metrics" + "github.com/samber/lo" +) + +var _ = Describe("Windows test", Label("unit"), func() { + DescribeTable("invalid Window struct", func(window metrics.Window) { + Expect(window.Validate()).NotTo(Succeed()) + }, + Entry("Empty window", metrics.Window{}), + Entry("missing long window", metrics.Window{ + ShortWindow: model.Duration(time.Minute), + ErrorBudgetPercent: 2, + }), + Entry("missing short window", metrics.Window{ + LongWindow: model.Duration(time.Hour), + ErrorBudgetPercent: 2, + }), + Entry("missing error budget", metrics.Window{ + LongWindow: model.Duration(time.Hour), + ShortWindow: model.Duration(time.Minute), + }), + ) + + DescribeTable("valid windows -- duration strings", func(window *metrics.MWMBWindows, windowRange []string) { + Expect(window.Validate()).To(Succeed()) + wr := window.WindowRange() + periodStrs := lo.Map(wr, func(w metrics.WindowMetadata, _ int) string { + return w.WindowDur.String() + }) + Expect(periodStrs).To(Equal(windowRange)) + }, + Entry("default window", metrics.WindowDefaults( + model.Duration(time.Hour*24*30), + ), []string{"5m", "30m", "1h", "2h", "6h", "6h", "1d", "3d", "30d"}), + Entry("custom window", metrics.GenerateMWMBWindows( + model.Duration(time.Hour*24*28), + model.Duration(time.Minute*5), + ), []string{"5m", "30m", "1h", "2h", "6h", "6h", "1d", "3d", "4w"}), + Entry("custom window short", metrics.GenerateMWMBWindows( + model.Duration(time.Hour*5), + model.Duration(time.Second*5), + ), []string{"5s", "30s", "1m", "2m", "6m", "6m", "24m", "1h12m", "5h"}), + Entry("normalizing period interval", + metrics.GenerateMWMBWindows( + model.Duration(time.Minute*72*10), + model.Duration(metrics.NormalizePeriodToBudgetingInterval(time.Minute*72*10)), + ), []string{"5s", "30s", "1m", "2m", "6m", "6m", "24m", "1h12m", "12h"}), + Entry("custom window very short", metrics.GenerateMWMBWindows( + model.Duration(time.Hour), + model.Duration(time.Second), + ), []string{"1s", "6s", "12s", "24s", "1m12s", "1m12s", "4m48s", "14m24s", "1h"}), + Entry("normalizing very short interval", + metrics.GenerateMWMBWindows( + model.Duration(aVeryExactDuration), + model.Duration(metrics.NormalizePeriodToBudgetingInterval(aVeryExactDuration)), + ), []string{"1s", "6s", "12s", "24s", "1m12s", "1m12s", "4m48s", "14m24s", "2h24m"}, + ), + ) + // TODO : burn rate factor tests +}) + +var ( + aVeryExactDuration = (time.Minute*14 + (time.Second * 24)) * 10 +) diff --git a/pkg/slo/backend/service.go b/pkg/slo/backend/service.go new file mode 100644 index 0000000000..4a75395aab --- /dev/null +++ b/pkg/slo/backend/service.go @@ -0,0 +1,13 @@ +package backend + +import ( + "context" + + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" +) + +type ServiceBackend interface { + ListServices(ctx context.Context, req *slov1.ListServicesRequest) (*slov1.ServiceList, error) + ListMetrics(ctx context.Context, req *slov1.ListMetricsRequest) (*slov1.MetricGroupList, error) + ListEvents(ctx context.Context, req *slov1.ListEventsRequest) (*slov1.EventList, error) +} diff --git a/pkg/slo/backend/slo.go b/pkg/slo/backend/slo.go new file mode 100644 index 0000000000..bfa46858bd --- /dev/null +++ b/pkg/slo/backend/slo.go @@ -0,0 +1,308 @@ +package backend + +import ( + "context" + "fmt" + "slices" + + "github.com/google/uuid" + corev1 "github.com/rancher/opni/pkg/apis/core/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/samber/lo" +) + +// TODO : tests +type SLO struct { + SloPeriod string + Objective float64 // 0 < x < 100 + Svc Service + GoodMetric Metric + TotalMetric Metric + IdLabels IdentificationLabels + UserLabels map[string]string + GoodEvents LabelPairs + TotalEvents LabelPairs +} + +type SLODatasource interface { + SLOStore + ServiceBackend + Precondition(ctx context.Context, clusterId *corev1.Reference) error +} + +type sLODatasourceImpl struct { + SLOStore + ServiceBackend + precondition func(ctx context.Context, clusterId *corev1.Reference) error +} + +func (s *sLODatasourceImpl) Precondition(ctx context.Context, clusterId *corev1.Reference) error { + return s.precondition(ctx, clusterId) +} + +func NewSLODatasource( + sloStore SLOStore, + serviceBackend ServiceBackend, + precondition func(ctx context.Context, clusterId *corev1.Reference) error, +) SLODatasource { + return &sLODatasourceImpl{ + SLOStore: sloStore, + ServiceBackend: serviceBackend, + precondition: precondition, + } +} + +func (s *SLO) Validate() error { + if s.SloPeriod == "" { + return fmt.Errorf("slo period is required") + } + if s.Objective <= 0 || s.Objective >= 100 { + return fmt.Errorf("slo object must be between 0 and 100 (exclusive)") + } + if s.Svc == "" { + return fmt.Errorf("slo service is required") + } + if s.GoodMetric == "" { + return fmt.Errorf("slo good metric is required") + } + if s.TotalMetric == "" { + return fmt.Errorf("slo total metric is required") + } + if len(s.GoodEvents) == 0 { + return fmt.Errorf("slo good events are required") + } + if err := s.IdLabels.Validate(); err != nil { + return err + } + return nil +} + +func (s *SLO) GetId() string { + return s.IdLabels[SLOUuid] +} + +func (s *SLO) SetId(id string) { + s.IdLabels[SLOUuid] = id +} + +func (s *SLO) GetName() string { + return s.IdLabels[SLOName] +} + +func (s *SLO) SetName(input string) { + s.IdLabels[SLOName] = input +} + +func (s *SLO) GetPeriod() string { + return s.SloPeriod +} + +func (s *SLO) GetObjective() float64 { + return s.Objective +} + +// MatchEventsOnMetric only applies when the good metric & total metric id is the same +func MatchEventsOnMetric(goodEvents, totalEvents []*slov1.Event) (good, total []*slov1.Event) { + goodAggregate := map[string][]string{} + for _, g := range goodEvents { + if g.Key == "" || g.Vals == nil { + continue + } + if _, ok := goodAggregate[g.Key]; !ok { + goodAggregate[g.Key] = []string{} + } + goodAggregate[g.Key] = append(goodAggregate[g.Key], g.Vals...) + } + totalAggregate := map[string][]string{} + for _, t := range totalEvents { + if t.Key == "" || t.Vals == nil { + continue + } + if _, ok := totalAggregate[t.Key]; !ok { + totalAggregate[t.Key] = []string{} + } + totalAggregate[t.Key] = append(totalAggregate[t.Key], t.Vals...) + } + for key, vals := range totalAggregate { + if _, ok := goodAggregate[key]; ok { + // total event filter defined on good and total so reconcile + totalAggregate[key] = LeftJoinSlice(goodAggregate[key], vals) + } else { + // total filter must also apply to good event filter + goodAggregate[key] = vals + } + } + retGoodEvent := lo.MapToSlice(goodAggregate, func(k string, v []string) *slov1.Event { + return &slov1.Event{ + Key: k, + Vals: v, + } + }) + retTotalEvent := lo.MapToSlice(totalAggregate, func(k string, v []string) *slov1.Event { + return &slov1.Event{ + Key: k, + Vals: v, + } + }) + + slices.SortFunc(retTotalEvent, func(a, b *slov1.Event) int { + if a.Key < b.Key { + return -1 + } else if a.Key > b.Key { + return 1 + } + return 0 + }) + + slices.SortFunc(retGoodEvent, func(a, b *slov1.Event) int { + if a.Key < b.Key { + return -1 + } else if a.Key > b.Key { + return 1 + } + return 0 + }) + return retGoodEvent, retTotalEvent +} + +func CreateSLORequestToStruct(c *slov1.CreateSLORequest) *SLO { + if c.Slo.GetGoodMetricName() == c.Slo.GetTotalMetricName() { + c.Slo.GoodEvents, c.Slo.TotalEvents = MatchEventsOnMetric(c.Slo.GoodEvents, c.Slo.TotalEvents) + } + reqSLO := c.Slo + userLabels := reqSLO.GetLabels() + sloLabels := map[string]string{} + for _, label := range userLabels { + sloLabels[label.GetName()] = "true" + } + goodEvents := []LabelPair{} + for _, goodEvent := range reqSLO.GetGoodEvents() { + goodEvents = append(goodEvents, LabelPair{ + Key: goodEvent.GetKey(), + Vals: goodEvent.GetVals(), + }) + } + totalEvents := []LabelPair{} + for _, totalEvent := range reqSLO.GetTotalEvents() { + totalEvents = append(totalEvents, LabelPair{ + Key: totalEvent.GetKey(), + Vals: totalEvent.GetVals(), + }) + } + return NewSLO( + reqSLO.GetName(), + reqSLO.GetSloPeriod(), + reqSLO.GetTarget().GetValue(), + Service(reqSLO.GetServiceId()), + Metric(reqSLO.GetGoodMetricName()), + Metric(reqSLO.GetTotalMetricName()), + sloLabels, + goodEvents, + totalEvents, + ) +} + +func NewSLO( + sloName string, + sloPeriod string, + objective float64, + svc Service, + goodMetric Metric, + totalMetric Metric, + userLabels map[string]string, + goodEvents []LabelPair, + totalEvents []LabelPair, +) *SLO { + newId := uuid.New().String() + ilabels := IdentificationLabels{SLOUuid: newId, SLOName: sloName, SLOService: string(svc)} + + return &SLO{ + Svc: svc, + SloPeriod: sloPeriod, + GoodMetric: goodMetric, + TotalMetric: totalMetric, + UserLabels: userLabels, + GoodEvents: goodEvents, + TotalEvents: totalEvents, + IdLabels: ilabels, + Objective: objective, + } +} + +func SLOFromId( + sloName string, + sloPeriod string, + objective float64, + svc Service, + goodMetric Metric, + totalMetric Metric, + userLabels map[string]string, + goodEvents []LabelPair, + totalEvents []LabelPair, + id string, +) *SLO { + ilabels := IdentificationLabels{SLOUuid: id, SLOName: sloName, SLOService: string(svc)} + + return &SLO{ + Svc: svc, + GoodMetric: goodMetric, + TotalMetric: totalMetric, + SloPeriod: sloPeriod, + UserLabels: userLabels, + GoodEvents: goodEvents, + TotalEvents: totalEvents, + IdLabels: ilabels, + Objective: objective, + } +} + +func SLODataToStruct(s *slov1.SLOData) *SLO { + reqSLO := s.SLO + if reqSLO.GetGoodMetricName() == reqSLO.GetTotalMetricName() { + reqSLO.GoodEvents, reqSLO.TotalEvents = MatchEventsOnMetric(reqSLO.GoodEvents, reqSLO.TotalEvents) + } + userLabels := reqSLO.GetLabels() + sloLabels := map[string]string{} + for _, label := range userLabels { + sloLabels[label.GetName()] = "true" + } + goodEvents := []LabelPair{} + for _, goodEvent := range reqSLO.GetGoodEvents() { + goodEvents = append(goodEvents, LabelPair{ + Key: goodEvent.GetKey(), + Vals: goodEvent.GetVals(), + }) + } + totalEvents := []LabelPair{} + for _, totalEvent := range reqSLO.GetTotalEvents() { + totalEvents = append(totalEvents, LabelPair{ + Key: totalEvent.GetKey(), + Vals: totalEvent.GetVals(), + }) + } + if s.Id == "" { + return NewSLO( + reqSLO.GetName(), + reqSLO.GetSloPeriod(), + reqSLO.GetTarget().GetValue(), + Service(reqSLO.GetServiceId()), + Metric(reqSLO.GetGoodMetricName()), + Metric(reqSLO.GetTotalMetricName()), + sloLabels, + goodEvents, + totalEvents, + ) + } + return SLOFromId( + reqSLO.GetName(), + reqSLO.GetSloPeriod(), + reqSLO.GetTarget().GetValue(), + Service(reqSLO.GetServiceId()), + Metric(reqSLO.GetGoodMetricName()), + Metric(reqSLO.GetTotalMetricName()), + sloLabels, + goodEvents, + totalEvents, + s.Id, + ) +} diff --git a/pkg/slo/backend/store.go b/pkg/slo/backend/store.go new file mode 100644 index 0000000000..c6f48b1d08 --- /dev/null +++ b/pkg/slo/backend/store.go @@ -0,0 +1,24 @@ +package backend + +import ( + "context" + + corev1 "github.com/rancher/opni/pkg/apis/core/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" +) + +type SLOStore interface { + Create(ctx context.Context, req *slov1.CreateSLORequest) (*corev1.Reference, error) + Update(ctx context.Context, incoming, existing *slov1.SLOData) (*slov1.SLOData, error) + Delete(ctx context.Context, existing *slov1.SLOData) error + + Clone(ctx context.Context, clone *slov1.SLOData) (*corev1.Reference, *slov1.SLOData, error) + MultiClusterClone( + ctx context.Context, + slo *slov1.SLOData, + toClusters []*corev1.Reference, + ) ([]*corev1.Reference, []*slov1.SLOData, []error) + + Status(ctx context.Context, existing *slov1.SLOData) (*slov1.SLOStatus, error) + Preview(ctx context.Context, s *slov1.CreateSLORequest) (*slov1.SLOPreviewResponse, error) +} diff --git a/pkg/slo/backend/util.go b/pkg/slo/backend/util.go new file mode 100644 index 0000000000..34d9d8b556 --- /dev/null +++ b/pkg/slo/backend/util.go @@ -0,0 +1,96 @@ +package backend + +import ( + "fmt" + "slices" + "sort" + "strings" + + "github.com/samber/lo" +) + +const ( + // id labels + SLOUuid = "slo_opni_id" + SLOService = "slo_opni_service" + SLOName = "slo_opni_name" +) + +type Service string +type Metric string + +type LabelPairs []LabelPair +type LabelPair struct { + Key string + Vals []string +} + +type IdentificationLabels map[string]string + +func (i IdentificationLabels) Validate() error { + if _, ok := i[SLOUuid]; !ok { + return fmt.Errorf("slo uuid label '%s' is required", SLOUuid) + } + if _, ok := i[SLOService]; !ok { + return fmt.Errorf("slo service label '%s' is required", SLOService) + } + if _, ok := i[SLOName]; !ok { + return fmt.Errorf("slo name label '%s' is required", SLOName) + } + return nil +} + +func (i IdentificationLabels) ToLabels() LabelPairs { + lbs := lo.MapToSlice(i, func(k string, v string) LabelPair { + return LabelPair{ + Key: k, + Vals: []string{v}, + } + }) + + slices.SortFunc(lbs, func(a, b LabelPair) int { + if a.Key < b.Key { + return -1 + } else if a.Key > b.Key { + return 1 + } + return 0 + }) + return lbs +} + +func (i IdentificationLabels) JoinOnPrometheus() string { + keys := lo.Keys(i) + sort.Strings(keys) + return strings.Join(keys, ", ") +} + +func (l LabelPairs) ConstructPrometheus() string { // kinda hacky & technically unoptimized but works + if len(l) == 0 { + return "" + } + filterStrings := []string{} + for _, labelPair := range l { + if labelPair.Key == "" || len(labelPair.Vals) == 0 { + continue + } + filterString := fmt.Sprintf("%s=~\"%s\"", labelPair.Key, strings.Join(labelPair.Vals, "|")) + filterStrings = append(filterStrings, filterString) + } + return strings.Join(filterStrings, ",") +} + +func LeftJoinSlice[T comparable](arr1, arr2 []T) []T { + result := make([]T, len(arr1)) + cache := map[T]struct{}{} + for i, v := range arr1 { + cache[v] = struct{}{} + result[i] = v + } + for _, v := range arr2 { + if _, ok := cache[v]; !ok { + result = append(result, v) + } + } + return result +} diff --git a/pkg/slo/backend/util_test.go b/pkg/slo/backend/util_test.go new file mode 100644 index 0000000000..57fd51ff03 --- /dev/null +++ b/pkg/slo/backend/util_test.go @@ -0,0 +1,60 @@ +package backend_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/rancher/opni/pkg/slo/backend" +) + +var _ = Describe("SLO util", Label("unit"), func() { + When("we use id labels util", func() { + Specify("it should convert to prometheus join on filters", func() { + i := backend.IdentificationLabels(map[string]string{ + "foo": "bar", + }) + Expect(i.JoinOnPrometheus()).To(Equal("foo")) + + i2 := backend.IdentificationLabels(map[string]string{ + "foo": "bar", + "baz": "qux", + }) + + Expect(i2.JoinOnPrometheus()).To(Equal("baz, foo")) + }) + + Specify("it should convert to SLO label pairs", func() { + i := backend.IdentificationLabels(map[string]string{ + "foo": "bar", + "baz": "qux", + }) + + Expect(i.ToLabels()).To(Equal(backend.LabelPairs{ + { + Key: "baz", + Vals: []string{"qux"}, + }, + { + Key: "foo", + Vals: []string{"bar"}, + }, + })) + }) + }) + + When("we use label pairs util", func() { + Specify("it should construct prometheus join on filters", func() { + lb := backend.LabelPairs{ + { + Key: "foo", + Vals: []string{"bar"}, + }, + { + Key: "baz", + Vals: []string{"qux", "qux2"}, + }, + } + filter := lb.ConstructPrometheus() + Expect(filter).To(Equal("foo=~\"bar\",baz=~\"qux|qux2\"")) + }) + }) +}) diff --git a/pkg/slo/query/collectors.go b/pkg/slo/query/collectors.go deleted file mode 100644 index a8a27f8f63..0000000000 --- a/pkg/slo/query/collectors.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Module for defining collectors and their good/bad events API. - -There are registered manually in preconfigured metrics builder in `def.go` - -There are registered automatically to an auto-instrumentation server -in `pkg/test/instrumentation.go` -*/ -package query - -import ( - "fmt" - "math/rand" - "net/http" - - "github.com/prometheus/client_golang/prometheus" -) - -var verbs = []string{"POST", "GET", "PUT", "DELETE"} -var goodCodes = []int{200, 201, 202} -var badCodes = []int{404, 429, 500, 502, 503} - -var ( - uptimeCollector *prometheus.GaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "uptime_good", - }, []string{"hostname", "ip", "job"}) - - uptimeGoodEvents http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - uptimeCollector.WithLabelValues(r.Host, r.RemoteAddr, MockTestServerName).Set(1) - } - uptimeBadEvents http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - uptimeCollector.WithLabelValues(r.Host, r.RemoteAddr, MockTestServerName).Set(0) - } - - availabilityCollector *prometheus.CounterVec = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "http_request_duration_seconds_count", - }, - []string{"code", "verb"}, - ) - availabilityGoodEvents http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - randomStatusCode := goodCodes[rand.Intn(len(goodCodes))] - randomVerb := verbs[rand.Intn(len(verbs))] - - availabilityCollector.WithLabelValues(fmt.Sprintf("%d", randomStatusCode), randomVerb).Inc() - } - - availabilityBadEvents http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - randomStatusCode := badCodes[rand.Intn(len(badCodes))] - randomVerb := verbs[rand.Intn(len(verbs))] - availabilityCollector.WithLabelValues(fmt.Sprintf("%d", randomStatusCode), randomVerb).Inc() - } - - latencyCollector *prometheus.HistogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Name: "http_request_duration_seconds_bucket", - }, []string{"hostname", "ip", "job"}) - - latencyGoodEvents http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - // request duration faster than 0.3s / 0.300 ms - randLatency := rand.Float64() * 0.29 - latencyCollector.WithLabelValues(r.Host, r.RemoteAddr, MockTestServerName).Observe(randLatency) - } - - latencyBadEvents http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - // request duration slower than 0.3s / 0.300 ms - randLatency := rand.Float64() + 0.30 - latencyCollector.WithLabelValues(r.Host, r.RemoteAddr, MockTestServerName).Observe(randLatency) - } -) diff --git a/pkg/slo/query/def.go b/pkg/slo/query/def.go deleted file mode 100644 index 3959b10c2a..0000000000 --- a/pkg/slo/query/def.go +++ /dev/null @@ -1,330 +0,0 @@ -package query - -/* -This module defines the datatypes & interfaces used to define SLO queries. -`AvailableQueries` contains the list of all preconfigured queries. - -Queries used by SLOs must follow a format like : - -totalQueryTempl = template.Must(template.New("").Parse(` - sum(rate({{.MetricId}}{job="{{.JobId}}"}[{{"{{.window}}"}}])) - `)) -goodQueryTempl = template.Must(template.New("").Parse(` - sum(rate({{.MetricId}}{job="{{.JobId}}", {{.Filter}}}[{{"{{.window}}"}}])) -`)) - -Must : - -1. Include a nested template with a {{.window}} for SLOs to fill in -2. the templates must only include information that can be filled with the `templateExecutor` struct. - Note: templates are intended to be filled with *api.Service protobuf definitions, so expect only that information will - be available, when SLOs are created at runtime -*/ - -import ( - "net/http" - "regexp" - "strings" - "text/template" - - "github.com/prometheus/client_golang/prometheus" - "github.com/rancher/opni/pkg/slo/shared" -) - -var ( - AvailableQueries = make(map[string]MetricQuery) - GetDownstreamMetricQueryTempl = template.Must(template.New("").Parse(` - group by(__name__)({__name__=~"{{.NameRegex}}"}) - `)) -) - -type templateExecutor struct { - MetricIdGood string - MetricIdTotal string - JobId string -} - -// must not contain spaces -const MockTestServerName = "MyServer" - -func init() { - // Names should be unique for each pre-configured query, as they are used as keys - // in the map - - //FIXME: doesn't turn into a prometheus range - uptimeSLOQuery := New(). - Name("uptime"). - GoodQuery( - NewQueryBuilder(). - Query(` - (sum(rate({{.MetricIdGood}}{job="{{.JobId}}"} == 1)))[{{"{{.window}}"}}] - `). - MetricFilter(`up`). - BuildRatio()). - TotalQuery( - NewQueryBuilder(). - Query(` - (sum(rate({{.MetricIdTotal}}{job="{{.JobId}}"})))[{{"{{.window}}"}}] - `). - MetricFilter(`up`).BuildRatio()). - Collector(uptimeCollector). - GoodEventGenerator(uptimeGoodEvents). - BadEventGenerator(uptimeBadEvents). - Description("Measures the uptime of a kubernetes service"). - Datasource(shared.MonitoringDatasource).Build() - AvailableQueries[uptimeSLOQuery.name] = &uptimeSLOQuery - - httpAvailabilitySLOQuery := New(). - Name("http-availability"). - GoodQuery( - NewQueryBuilder(). - Query(` - (sum(rate({{.MetricIdGood}}{job="{{.JobId}}",code=~"(2..|3..)"}[{{"{{.window}}"}}]))) - `). - MetricFilter(`.*http_request_duration_seconds_count`). - BuildRatio()). - TotalQuery( - NewQueryBuilder(). - Query(` - (sum(rate({{.MetricIdTotal}}{job="{{.JobId}}"}[{{"{{.window}}"}}]))) - `). - MetricFilter(`.*http_request_duration_seconds_count`).BuildRatio()). - Collector(availabilityCollector). - GoodEventGenerator(availabilityGoodEvents). - BadEventGenerator(availabilityBadEvents). - Description(`Measures the availability of a kubernetes service using http status codes. - Codes 2XX and 3XX are considered as available.`). - Datasource(shared.MonitoringDatasource).Build() - AvailableQueries[httpAvailabilitySLOQuery.name] = &httpAvailabilitySLOQuery - - httpResponseTimeSLOQuery := New(). - Name("http-latency"). - GoodQuery( - NewQueryBuilder(). - Query(` - sum(rate({{.MetricIdGood}}{job="{{.JobId}},"le="0.3",verb!="WATCH"}[{{"{{.window}}"}}])) - `). - MetricFilter(`.*http_request_duration_seconds_bucket`). - BuildHistogram()). - TotalQuery( - NewQueryBuilder(). - Query(` - sum(rate({{.MetricIdTotal}}{job="{{.JobId}}",verb!="WATCH"}[{{"{{.window}}"}}])) - `). - MetricFilter(`.*http_request_duration_seconds_count`).BuildRatio()). - Collector(latencyCollector). - GoodEventGenerator(latencyGoodEvents). - BadEventGenerator(latencyBadEvents). - Description(`Quantifies the latency of http requests made against a kubernetes service - by classifying them as good (<=300ms) or bad(>=300ms)`). - Datasource(shared.MonitoringDatasource).Build() - AvailableQueries[httpResponseTimeSLOQuery.name] = &httpResponseTimeSLOQuery -} - -type matcher func([]string) string - -type SLOQueryResult struct { - GoodQuery string - TotalQuery string -} - -type ServiceInfo interface { - GetMetricIdGood() string - GetMetricIdTotal() string - GetJobId() string -} - -type MetricQuery interface { - // User facing name of the pre-confured metric - Name() string - // User-facing description of the pre-configured metric - Description() string - // Each metric has a unique opni datasource (monitoring vs logging) by which it is filtered by - Datasource() string - Construct(ServiceInfo) (*SLOQueryResult, error) - // Some metrics will have different labels for metrics, so handle them independently - GetGoodQuery() Query - GetTotalQuery() Query - - // Auto-instrumentation server methods - - GetCollector() prometheus.Collector - GetGoodEventGenerator() func(w http.ResponseWriter, r *http.Request) - GetBadEventGenerator() func(w http.ResponseWriter, r *http.Request) -} - -type Query interface { - FillQueryTemplate(info templateExecutor) (string, error) - GetMetricFilter() string - Validate() error - IsRatio() bool - BestMatch([]string) string - IsHistogram() bool - Construct(ServiceInfo) (string, error) -} - -type QueryBuilder interface { - Query(string) QueryBuilder - MetricFilter(string) QueryBuilder - Matcher(*matcher) QueryBuilder - BuildRatio() RatioQuery - BuildHistogram() HistogramQuery -} - -type queryBuilder struct { - query template.Template - filter regexp.Regexp - matcher matcher -} - -func NewQueryBuilder() QueryBuilder { - return queryBuilder{} -} - -func (q queryBuilder) Query(query string) QueryBuilder { - tmpl := template.Must(template.New("").Parse(strings.TrimSpace(query))) - q.query = *tmpl - return q -} - -func (q queryBuilder) MetricFilter(filter string) QueryBuilder { - regex := regexp.MustCompile(strings.TrimSpace(filter)) - q.filter = *regex - return q -} - -// defaults to `MatchMinLength` -func (q queryBuilder) Matcher(matcher *matcher) QueryBuilder { - if matcher == nil { - q.matcher = MatchMinLength - } else { - q.matcher = *matcher - } - return q -} - -func (q queryBuilder) BuildRatio() RatioQuery { - if q.matcher == nil { - q.matcher = MatchMinLength - } - r := RatioQuery{ - query: q.query, - metricFilter: q.filter, - matcher: q.matcher, - } - err := r.Validate() - if err != nil { - panic(err) - } - return r -} - -func (q queryBuilder) BuildHistogram() HistogramQuery { - if q.matcher == nil { - q.matcher = MatchMinLength - } - h := HistogramQuery{ - query: q.query, - metricFilter: q.filter, - matcher: q.matcher, - } - err := h.Validate() - if err != nil { - panic(err) - } - return h -} - -type SloQueryBuilder interface { - Name(name string) SloQueryBuilder - GoodQuery(q Query) SloQueryBuilder - TotalQuery(q Query) SloQueryBuilder - Description(description string) SloQueryBuilder - Datasource(datasource string) SloQueryBuilder - Collector(prometheus.Collector) SloQueryBuilder - GoodEventGenerator(goodEvents func(w http.ResponseWriter, r *http.Request)) SloQueryBuilder - BadEventGenerator(badEvents func(w http.ResponseWriter, r *http.Request)) SloQueryBuilder - Build() PrometheusQueryImpl -} - -type sloQueryBuilder struct { - name string - metricFilter regexp.Regexp - goodQuery Query - totalQuery Query - description string - datasource string - collector prometheus.Collector - totalCollector prometheus.Collector - goodEventsGen func(w http.ResponseWriter, r *http.Request) - badEventsGen func(w http.ResponseWriter, r *http.Request) -} - -func New() SloQueryBuilder { - return sloQueryBuilder{} -} - -func (s sloQueryBuilder) Name(name string) SloQueryBuilder { - s.name = name - return s -} - -func (s sloQueryBuilder) GoodQuery(q Query) SloQueryBuilder { - s.goodQuery = q - return s -} - -func (s sloQueryBuilder) TotalQuery(q Query) SloQueryBuilder { - s.totalQuery = q - return s -} - -func (s sloQueryBuilder) Description(description string) SloQueryBuilder { - s.description = description - return s -} - -func (s sloQueryBuilder) Datasource(datasource string) SloQueryBuilder { - s.datasource = datasource - return s -} - -func (s sloQueryBuilder) Collector( - collector prometheus.Collector) SloQueryBuilder { - s.collector = collector - return s -} - -func (s sloQueryBuilder) GoodEventGenerator( - goodEvents func(w http.ResponseWriter, r *http.Request), -) SloQueryBuilder { - s.goodEventsGen = goodEvents - return s -} - -func (s sloQueryBuilder) BadEventGenerator( - badEvents func(w http.ResponseWriter, r *http.Request), -) SloQueryBuilder { - s.badEventsGen = badEvents - return s -} - -func (s sloQueryBuilder) Build() PrometheusQueryImpl { - //TODO figure out how to validate at compile time - if s.collector == nil { - panic("auto-instrumentation collectors must be set") - } - if s.goodEventsGen == nil || s.badEventsGen == nil { - panic("auto-instrumentation events must be set") - } - return PrometheusQueryImpl{ - name: s.name, - datasource: s.datasource, - description: s.description, - GoodQuery: s.goodQuery, - TotalQuery: s.totalQuery, - collector: s.collector, - goodEventsGen: s.goodEventsGen, - badEventsGen: s.badEventsGen, - } -} diff --git a/pkg/slo/query/histogram.go b/pkg/slo/query/histogram.go deleted file mode 100644 index ab2690d3e4..0000000000 --- a/pkg/slo/query/histogram.go +++ /dev/null @@ -1,71 +0,0 @@ -package query - -import ( - "bytes" - "net/http" - "regexp" - "strings" - "text/template" - - "github.com/prometheus/client_golang/prometheus" -) - -type HistogramQuery struct { - query template.Template - metricFilter regexp.Regexp - matcher matcher - goodCollector *prometheus.Collector - totalCollector *prometheus.Collector - goodEvents func(w http.ResponseWriter, r *http.Request) - totalEvents func(w http.ResponseWriter, r *http.Request) -} - -func (hq HistogramQuery) GetGoodCollector() prometheus.Collector { - return *hq.goodCollector -} - -func (hq HistogramQuery) GetTotalCollector() prometheus.Collector { - return *hq.totalCollector -} -func (hq HistogramQuery) GetGoodEvents() func(w http.ResponseWriter, r *http.Request) { - return hq.goodEvents -} -func (hq HistogramQuery) GetTotalEvents() func(w http.ResponseWriter, r *http.Request) { - return hq.totalEvents -} - -func (hq HistogramQuery) FillQueryTemplate(info templateExecutor) (string, error) { - var query bytes.Buffer - if err := hq.query.Execute(&query, info); err != nil { - return "", err - } - return strings.TrimSpace(query.String()), nil -} - -func (hq HistogramQuery) GetMetricFilter() string { - return hq.metricFilter.String() -} - -func (hq HistogramQuery) Validate() error { - return validatePromQl(hq.query) -} - -func (hq HistogramQuery) IsRatio() bool { - return false -} - -func (hq HistogramQuery) IsHistogram() bool { - return true -} - -func (hq HistogramQuery) Construct(serv ServiceInfo) (string, error) { - return hq.FillQueryTemplate(templateExecutor{ - MetricIdGood: serv.GetMetricIdGood(), - MetricIdTotal: serv.GetMetricIdTotal(), - JobId: serv.GetJobId(), - }) -} - -func (hq HistogramQuery) BestMatch(in []string) string { - return hq.matcher(in) -} diff --git a/pkg/slo/query/matchers.go b/pkg/slo/query/matchers.go deleted file mode 100644 index ddd85ea22d..0000000000 --- a/pkg/slo/query/matchers.go +++ /dev/null @@ -1,23 +0,0 @@ -package query - -import "math" - -/* -This module defines matcher functions for finding the best match for the -discovered downstream metrics -*/ - -func MatchMinLength(metrics []string) string { - min := math.MaxInt // largest int - metricId := "" - for _, m := range metrics { - if m == "" { - continue - } - if len(m) < min { - min = len(m) - metricId = m - } - } - return metricId -} diff --git a/pkg/slo/query/opensearch.go b/pkg/slo/query/opensearch.go deleted file mode 100644 index 0100a189cb..0000000000 --- a/pkg/slo/query/opensearch.go +++ /dev/null @@ -1,6 +0,0 @@ -package query - -/* -Make sure pre-configured metrics can be exported as valid opensearch / monitors -to the SLO api. -*/ diff --git a/pkg/slo/query/prometheus.go b/pkg/slo/query/prometheus.go deleted file mode 100644 index 4da7679c15..0000000000 --- a/pkg/slo/query/prometheus.go +++ /dev/null @@ -1,69 +0,0 @@ -package query - -/* -Make sure pre-configured metrics can be exported as valid prometheus/ promql -to the SLO api. -*/ - -import ( - "net/http" - "regexp" - - "github.com/prometheus/client_golang/prometheus" -) - -type PrometheusQueryImpl struct { - name string - GoodQuery Query - TotalQuery Query - LabelRegex regexp.Regexp - datasource string - description string - collector prometheus.Collector - goodEventsGen func(w http.ResponseWriter, r *http.Request) - badEventsGen func(w http.ResponseWriter, r *http.Request) -} - -// The actual metricId and window are only known at SLO creation time -func (p *PrometheusQueryImpl) Construct(service ServiceInfo) (*SLOQueryResult, error) { - goodQueryStr, err := p.GoodQuery.Construct(service) - if err != nil { - return nil, err - } - totalQueryStr, err := p.TotalQuery.Construct(service) - if err != nil { - return nil, err - } - return &SLOQueryResult{GoodQuery: goodQueryStr, TotalQuery: totalQueryStr}, nil -} - -func (p *PrometheusQueryImpl) Datasource() string { - return p.datasource -} - -func (p *PrometheusQueryImpl) Description() string { - return p.description -} - -func (p *PrometheusQueryImpl) Name() string { - return p.name -} - -func (p *PrometheusQueryImpl) GetGoodQuery() Query { - return p.GoodQuery -} - -func (p *PrometheusQueryImpl) GetTotalQuery() Query { - return p.TotalQuery -} - -func (p *PrometheusQueryImpl) GetCollector() prometheus.Collector { - return p.collector -} - -func (p *PrometheusQueryImpl) GetGoodEventGenerator() func(w http.ResponseWriter, r *http.Request) { - return p.goodEventsGen -} -func (p *PrometheusQueryImpl) GetBadEventGenerator() func(w http.ResponseWriter, r *http.Request) { - return p.badEventsGen -} diff --git a/pkg/slo/query/query_test.go b/pkg/slo/query/query_test.go deleted file mode 100644 index 170f9c48f2..0000000000 --- a/pkg/slo/query/query_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package query_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/rancher/opni/pkg/slo/query" - api "github.com/rancher/opni/plugins/slo/apis/slo" -) - -func constructionShouldSucceed(q *query.SLOQueryResult, err error) { - Expect(err).To(Succeed()) - Expect(q).To(Not(BeNil())) - Expect(q.GoodQuery).To(Not(BeNil())) - Expect(q.TotalQuery).To(Not(BeNil())) -} - -var _ = Describe("Converting ServiceLevelObjective Messages to Prometheus Rules", Ordered, Label("unit", "slow"), func() { - // ctx := context.Background() - - validMetricQuery := func(query query.MetricQuery) error { - return nil - } - - When("We define pre-configured metrics in def.go", func() { - It("Should produce valid templates and regex at compile time", func() { - Expect(query.AvailableQueries).ToNot(HaveLen(0)) // > 0 available pre configured metrics - }) - - It("Query implementations should implement the interfaces required", func() { - q := &query.PrometheusQueryImpl{} - validMetricQuery(q) - }) - }) - - When("We use matchers", func() { - It("Should use a MatchMinLength to match the shortest metric", func() { - m := query.MatchMinLength - discoveredMetrics := []string{ - "alertmanager_http_request_duration_seconds_bucket", - "alertmanager_http_request_duration_seconds_count", - "alertmanager_http_request_duration_seconds_sum", - } - Expect(m(discoveredMetrics)).To(Equal("alertmanager_http_request_duration_seconds_sum")) - }) - }) - - When("We construct metrics via Service defintions", func() { - It("Should fill in the templates & return a response struct", func() { - uptime, ok := query.AvailableQueries["uptime"] - Expect(ok).To(BeTrue()) - service := &api.ServiceInfo{ - JobId: "prometheus", - MetricName: "uptime", - MetricIdGood: "up", - MetricIdTotal: "up", - ClusterId: "agent", - } - respUp, err := uptime.Construct(service) - constructionShouldSucceed(respUp, err) - Expect(respUp.GoodQuery).To(Equal("(sum(rate(up{job=\"prometheus\"} == 1)))[{{.window}}]")) - Expect(respUp.TotalQuery).To(Equal("(sum(rate(up{job=\"prometheus\"})))[{{.window}}]")) - - latency, ok := query.AvailableQueries["http-latency"] - Expect(ok).To(BeTrue()) - respLa, err := latency.Construct(service) - constructionShouldSucceed(respLa, err) - - availability, ok := query.AvailableQueries["http-availability"] - Expect(ok).To(BeTrue()) - respAv, err := availability.Construct(service) - constructionShouldSucceed(respAv, err) - - }) - }) -}) diff --git a/pkg/slo/query/ratio.go b/pkg/slo/query/ratio.go deleted file mode 100644 index 0cb2d8b55a..0000000000 --- a/pkg/slo/query/ratio.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Contains implementation details for slo ratio query metrics -*/ - -package query - -import ( - "bytes" - "net/http" - "regexp" - "strings" - "text/template" - - "github.com/prometheus/client_golang/prometheus" -) - -type RatioQuery struct { - query template.Template - metricFilter regexp.Regexp // how to assign the metric to the query - matcher matcher - goodCollector *prometheus.Collector - totalCollector *prometheus.Collector - goodEvents func(w http.ResponseWriter, r *http.Request) - totalEvents func(w http.ResponseWriter, r *http.Request) -} - -func (rq RatioQuery) GetGoodCollector() prometheus.Collector { - return *rq.goodCollector -} - -func (rq RatioQuery) GetTotalCollector() prometheus.Collector { - return *rq.totalCollector -} -func (rq RatioQuery) GetGoodEvents() func(w http.ResponseWriter, r *http.Request) { - return rq.goodEvents -} -func (rq RatioQuery) GetTotalEvents() func(w http.ResponseWriter, r *http.Request) { - return rq.totalEvents -} - -func (rq RatioQuery) FillQueryTemplate(info templateExecutor) (string, error) { - var query bytes.Buffer - if err := rq.query.Execute(&query, info); err != nil { - return "", err - } - return strings.TrimSpace(query.String()), nil -} - -func (rq RatioQuery) GetMetricFilter() string { - return rq.metricFilter.String() -} - -func (rq RatioQuery) Validate() error { - return validatePromQl(rq.query) -} - -func (rq RatioQuery) IsRatio() bool { - return true -} - -func (rq RatioQuery) IsHistogram() bool { - return false -} - -func (rq RatioQuery) Construct(serv ServiceInfo) (string, error) { - return rq.FillQueryTemplate(templateExecutor{ - MetricIdGood: serv.GetMetricIdGood(), - MetricIdTotal: serv.GetMetricIdTotal(), - JobId: serv.GetJobId(), - }) -} - -func (rq RatioQuery) BestMatch(in []string) string { - return rq.matcher(in) -} diff --git a/pkg/slo/query/validate.go b/pkg/slo/query/validate.go deleted file mode 100644 index 925cb71215..0000000000 --- a/pkg/slo/query/validate.go +++ /dev/null @@ -1,36 +0,0 @@ -package query - -import ( - "text/template" - // prommodel "github.com/prometheus/common/model" -) - -// fill in query templates with mock information -var mockTemplateExecutor = templateExecutor{ - MetricIdGood: "metric_id_good", - MetricIdTotal: "metric_id_total", - JobId: "job_id", -} - -type mockSlothInfo struct { - window string -} - -// fill in sloth IR with mock information -var requiredSlothInfo = mockSlothInfo{ - window: "1h", -} - -func validatePromQl(_ template.Template) error { - // var intermediate bytes.Buffer - // queryTpl.Execute(&intermediate, mockTemplateExecutor) - // // slothTmpl := template.Must(template.New("").Parse(intermediate.String())) - // // var q bytes.Buffer - // // slothTmpl.Execute(&q, requiredSlothInfo) - // q := strings.ReplaceAll(intermediate.String(), "[{{.window}}]", "") - //_, err := promqlparser.ParseExpr(q) - // if err != nil { - // return err - // } - return nil -} diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 6e36b624f1..5cc8f4faba 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -35,11 +35,8 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nkeys" "github.com/onsi/ginkgo/v2" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rancher/opni/pkg/agent" agentv2 "github.com/rancher/opni/pkg/agent/v2" - "github.com/rancher/opni/pkg/alerting/metrics/naming" "github.com/rancher/opni/pkg/alerting/shared" alertingv1 "github.com/rancher/opni/pkg/apis/alerting/v1" corev1 "github.com/rancher/opni/pkg/apis/core/v1" @@ -61,7 +58,6 @@ import ( "github.com/rancher/opni/pkg/plugins" "github.com/rancher/opni/pkg/plugins/hooks" pluginmeta "github.com/rancher/opni/pkg/plugins/meta" - "github.com/rancher/opni/pkg/slo/query" "github.com/rancher/opni/pkg/test/freeport" mock_ident "github.com/rancher/opni/pkg/test/mock/ident" "github.com/rancher/opni/pkg/test/testdata" @@ -1097,141 +1093,6 @@ func (e *Environment) StartOTELCollectorContext(ctx waitctx.PermissiveContext, o return nil } -// Starts a server that exposes Prometheus metrics -// -// Returns port number of the server & a channel that shutdowns the server -func (e *Environment) StartInstrumentationServer() (int, chan struct{}) { - // lg := e.logger - port := freeport.GetFreePort() - - mux := http.NewServeMux() - reg := prometheus.NewRegistry() - - for queryName, queryObj := range query.AvailableQueries { - // register each prometheus collector - reg.MustRegister(queryObj.GetCollector()) - - // create an endpoint simulating good events - mux.HandleFunc(fmt.Sprintf("/%s/%s", queryName, "good"), queryObj.GetGoodEventGenerator()) - // create an endpoint simulating bad events - mux.HandleFunc(fmt.Sprintf("/%s/%s", queryName, "bad"), queryObj.GetBadEventGenerator()) - - } - // expose prometheus metrics - - mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{ - Registry: reg, - })) - - autoInstrumentationServer := &http.Server{ - Addr: fmt.Sprintf("127.0.0.1:%d", port), - Handler: mux, - ReadTimeout: 1 * time.Second, - WriteTimeout: 1 * time.Second, - MaxHeaderBytes: 1 << 20, - } - done := make(chan struct{}) - waitctx.Restrictive.Go(e.ctx, func() { - go func() { - err := autoInstrumentationServer.ListenAndServe() - if !errors.Is(err, http.ErrServerClosed) { - panic(err) - } - }() - defer autoInstrumentationServer.Shutdown(context.Background()) - select { - case <-e.ctx.Done(): - case <-done: - } - }) - return port, done -} - -func (e *Environment) StartMockKubernetesMetricServer() (port int) { - port = freeport.GetFreePort() - - mux := http.NewServeMux() - reg := prometheus.NewRegistry() - - registeredCollectors := map[string]*prometheus.GaugeVec{} - - kubeMetricsIsDefined := prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: naming.KubeMetricsIsDefinedMetricName, - }, []string{"namespace"}) - - kubeMetricsIsDefined.WithLabelValues("kube-system").Set(1) - - setObjHandler := func(w http.ResponseWriter, r *http.Request) { - collectorWriteSync.Lock() - defer collectorWriteSync.Unlock() - objType := r.URL.Query().Get("obj") - name := r.URL.Query().Get("name") - namespace := r.URL.Query().Get("namespace") - phase := r.URL.Query().Get("phase") - uid := r.URL.Query().Get("uid") - - accessedState := false - var b bytes.Buffer - err := naming.KubeObjMetricCreator.Execute(&b, map[string]string{ - "ObjType": objType, - }) - if err != nil { - panic(err) - } - var newOrExistingKubeObjStateCollector *prometheus.GaugeVec - if _, ok := registeredCollectors[b.String()]; !ok { - newOrExistingKubeObjStateCollector = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: b.String(), - }, []string{objType, "namespace", "phase", "uid"}) - registeredCollectors[b.String()] = newOrExistingKubeObjStateCollector - reg.MustRegister(newOrExistingKubeObjStateCollector) - } else { - newOrExistingKubeObjStateCollector = registeredCollectors[b.String()] - } - - for _, validPhase := range shared.KubeStates { - if phase == validPhase { - accessedState = true - newOrExistingKubeObjStateCollector.WithLabelValues(name, namespace, validPhase, uid).Set(1) - } else { - newOrExistingKubeObjStateCollector.WithLabelValues(name, namespace, validPhase, uid).Set(0) - } - } - if accessedState == false { - panic(fmt.Sprintf("Set state for kube metrics api must be one of %s", strings.Join(shared.KubeStates, ","))) - } - } - - reg.MustRegister(kubeMetricsIsDefined) - - mux.HandleFunc("/set", setObjHandler) - mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{ - Registry: reg, - })) - - autoKubernetesMetricsServer := &http.Server{ - Addr: fmt.Sprintf("127.0.0.1:%d", port), - Handler: mux, - ReadTimeout: 1 * time.Second, - WriteTimeout: 1 * time.Second, - MaxHeaderBytes: 1 << 20, - } - waitctx.Permissive.Go(e.ctx, func() { - go func() { - err := autoKubernetesMetricsServer.ListenAndServe() - if err != http.ErrServerClosed { - panic(err) - } - }() - defer autoKubernetesMetricsServer.Shutdown(context.Background()) - select { - case <-e.ctx.Done(): - } - }) - - return port -} - func (e *Environment) StartAgentDisconnectServer() { setDisconnect := func(w http.ResponseWriter, r *http.Request) { agentListMu.Lock() diff --git a/pkg/test/mock/cortexadmin/cortexadmin.go b/pkg/test/mock/cortexadmin/cortexadmin.go new file mode 100644 index 0000000000..3457d5ecbb --- /dev/null +++ b/pkg/test/mock/cortexadmin/cortexadmin.go @@ -0,0 +1,633 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: plugins/metrics/apis/cortexadmin/cortexadmin_grpc.pb.go + +// Package mock_cortexadmin is a generated GoMock package. +package mock_cortexadmin + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + cortexadmin "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" + grpc "google.golang.org/grpc" + emptypb "google.golang.org/protobuf/types/known/emptypb" +) + +// MockCortexAdminClient is a mock of CortexAdminClient interface. +type MockCortexAdminClient struct { + ctrl *gomock.Controller + recorder *MockCortexAdminClientMockRecorder +} + +// MockCortexAdminClientMockRecorder is the mock recorder for MockCortexAdminClient. +type MockCortexAdminClientMockRecorder struct { + mock *MockCortexAdminClient +} + +// NewMockCortexAdminClient creates a new mock instance. +func NewMockCortexAdminClient(ctrl *gomock.Controller) *MockCortexAdminClient { + mock := &MockCortexAdminClient{ctrl: ctrl} + mock.recorder = &MockCortexAdminClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCortexAdminClient) EXPECT() *MockCortexAdminClientMockRecorder { + return m.recorder +} + +// AllUserStats mocks base method. +func (m *MockCortexAdminClient) AllUserStats(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*cortexadmin.UserIDStatsList, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "AllUserStats", varargs...) + ret0, _ := ret[0].(*cortexadmin.UserIDStatsList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AllUserStats indicates an expected call of AllUserStats. +func (mr *MockCortexAdminClientMockRecorder) AllUserStats(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllUserStats", reflect.TypeOf((*MockCortexAdminClient)(nil).AllUserStats), varargs...) +} + +// DeleteRule mocks base method. +func (m *MockCortexAdminClient) DeleteRule(ctx context.Context, in *cortexadmin.DeleteRuleRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DeleteRule", varargs...) + ret0, _ := ret[0].(*emptypb.Empty) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteRule indicates an expected call of DeleteRule. +func (mr *MockCortexAdminClientMockRecorder) DeleteRule(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRule", reflect.TypeOf((*MockCortexAdminClient)(nil).DeleteRule), varargs...) +} + +// ExtractRawSeries mocks base method. +func (m *MockCortexAdminClient) ExtractRawSeries(ctx context.Context, in *cortexadmin.MatcherRequest, opts ...grpc.CallOption) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ExtractRawSeries", varargs...) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ExtractRawSeries indicates an expected call of ExtractRawSeries. +func (mr *MockCortexAdminClientMockRecorder) ExtractRawSeries(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExtractRawSeries", reflect.TypeOf((*MockCortexAdminClient)(nil).ExtractRawSeries), varargs...) +} + +// FlushBlocks mocks base method. +func (m *MockCortexAdminClient) FlushBlocks(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*emptypb.Empty, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "FlushBlocks", varargs...) + ret0, _ := ret[0].(*emptypb.Empty) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FlushBlocks indicates an expected call of FlushBlocks. +func (mr *MockCortexAdminClientMockRecorder) FlushBlocks(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FlushBlocks", reflect.TypeOf((*MockCortexAdminClient)(nil).FlushBlocks), varargs...) +} + +// GetCortexConfig mocks base method. +func (m *MockCortexAdminClient) GetCortexConfig(ctx context.Context, in *cortexadmin.ConfigRequest, opts ...grpc.CallOption) (*cortexadmin.ConfigResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetCortexConfig", varargs...) + ret0, _ := ret[0].(*cortexadmin.ConfigResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCortexConfig indicates an expected call of GetCortexConfig. +func (mr *MockCortexAdminClientMockRecorder) GetCortexConfig(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCortexConfig", reflect.TypeOf((*MockCortexAdminClient)(nil).GetCortexConfig), varargs...) +} + +// GetCortexStatus mocks base method. +func (m *MockCortexAdminClient) GetCortexStatus(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*cortexadmin.CortexStatus, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetCortexStatus", varargs...) + ret0, _ := ret[0].(*cortexadmin.CortexStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCortexStatus indicates an expected call of GetCortexStatus. +func (mr *MockCortexAdminClientMockRecorder) GetCortexStatus(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCortexStatus", reflect.TypeOf((*MockCortexAdminClient)(nil).GetCortexStatus), varargs...) +} + +// GetMetricLabelSets mocks base method. +func (m *MockCortexAdminClient) GetMetricLabelSets(ctx context.Context, in *cortexadmin.LabelRequest, opts ...grpc.CallOption) (*cortexadmin.MetricLabels, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetMetricLabelSets", varargs...) + ret0, _ := ret[0].(*cortexadmin.MetricLabels) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMetricLabelSets indicates an expected call of GetMetricLabelSets. +func (mr *MockCortexAdminClientMockRecorder) GetMetricLabelSets(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetricLabelSets", reflect.TypeOf((*MockCortexAdminClient)(nil).GetMetricLabelSets), varargs...) +} + +// GetMetricMetadata mocks base method. +func (m *MockCortexAdminClient) GetMetricMetadata(ctx context.Context, in *cortexadmin.MetricMetadataRequest, opts ...grpc.CallOption) (*cortexadmin.MetricMetadata, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetMetricMetadata", varargs...) + ret0, _ := ret[0].(*cortexadmin.MetricMetadata) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMetricMetadata indicates an expected call of GetMetricMetadata. +func (mr *MockCortexAdminClientMockRecorder) GetMetricMetadata(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetricMetadata", reflect.TypeOf((*MockCortexAdminClient)(nil).GetMetricMetadata), varargs...) +} + +// GetRule mocks base method. +func (m *MockCortexAdminClient) GetRule(ctx context.Context, in *cortexadmin.GetRuleRequest, opts ...grpc.CallOption) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetRule", varargs...) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRule indicates an expected call of GetRule. +func (mr *MockCortexAdminClientMockRecorder) GetRule(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRule", reflect.TypeOf((*MockCortexAdminClient)(nil).GetRule), varargs...) +} + +// GetSeriesMetrics mocks base method. +func (m *MockCortexAdminClient) GetSeriesMetrics(ctx context.Context, in *cortexadmin.SeriesRequest, opts ...grpc.CallOption) (*cortexadmin.SeriesInfoList, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetSeriesMetrics", varargs...) + ret0, _ := ret[0].(*cortexadmin.SeriesInfoList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSeriesMetrics indicates an expected call of GetSeriesMetrics. +func (mr *MockCortexAdminClientMockRecorder) GetSeriesMetrics(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSeriesMetrics", reflect.TypeOf((*MockCortexAdminClient)(nil).GetSeriesMetrics), varargs...) +} + +// ListRules mocks base method. +func (m *MockCortexAdminClient) ListRules(ctx context.Context, in *cortexadmin.ListRulesRequest, opts ...grpc.CallOption) (*cortexadmin.ListRulesResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListRules", varargs...) + ret0, _ := ret[0].(*cortexadmin.ListRulesResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRules indicates an expected call of ListRules. +func (mr *MockCortexAdminClientMockRecorder) ListRules(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRules", reflect.TypeOf((*MockCortexAdminClient)(nil).ListRules), varargs...) +} + +// LoadRules mocks base method. +func (m *MockCortexAdminClient) LoadRules(ctx context.Context, in *cortexadmin.LoadRuleRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "LoadRules", varargs...) + ret0, _ := ret[0].(*emptypb.Empty) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LoadRules indicates an expected call of LoadRules. +func (mr *MockCortexAdminClientMockRecorder) LoadRules(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadRules", reflect.TypeOf((*MockCortexAdminClient)(nil).LoadRules), varargs...) +} + +// Query mocks base method. +func (m *MockCortexAdminClient) Query(ctx context.Context, in *cortexadmin.QueryRequest, opts ...grpc.CallOption) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Query", varargs...) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Query indicates an expected call of Query. +func (mr *MockCortexAdminClientMockRecorder) Query(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Query", reflect.TypeOf((*MockCortexAdminClient)(nil).Query), varargs...) +} + +// QueryRange mocks base method. +func (m *MockCortexAdminClient) QueryRange(ctx context.Context, in *cortexadmin.QueryRangeRequest, opts ...grpc.CallOption) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "QueryRange", varargs...) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QueryRange indicates an expected call of QueryRange. +func (mr *MockCortexAdminClientMockRecorder) QueryRange(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryRange", reflect.TypeOf((*MockCortexAdminClient)(nil).QueryRange), varargs...) +} + +// WriteMetrics mocks base method. +func (m *MockCortexAdminClient) WriteMetrics(ctx context.Context, in *cortexadmin.WriteRequest, opts ...grpc.CallOption) (*cortexadmin.WriteResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "WriteMetrics", varargs...) + ret0, _ := ret[0].(*cortexadmin.WriteResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// WriteMetrics indicates an expected call of WriteMetrics. +func (mr *MockCortexAdminClientMockRecorder) WriteMetrics(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteMetrics", reflect.TypeOf((*MockCortexAdminClient)(nil).WriteMetrics), varargs...) +} + +// MockCortexAdminServer is a mock of CortexAdminServer interface. +type MockCortexAdminServer struct { + ctrl *gomock.Controller + recorder *MockCortexAdminServerMockRecorder +} + +// MockCortexAdminServerMockRecorder is the mock recorder for MockCortexAdminServer. +type MockCortexAdminServerMockRecorder struct { + mock *MockCortexAdminServer +} + +// NewMockCortexAdminServer creates a new mock instance. +func NewMockCortexAdminServer(ctrl *gomock.Controller) *MockCortexAdminServer { + mock := &MockCortexAdminServer{ctrl: ctrl} + mock.recorder = &MockCortexAdminServerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCortexAdminServer) EXPECT() *MockCortexAdminServerMockRecorder { + return m.recorder +} + +// AllUserStats mocks base method. +func (m *MockCortexAdminServer) AllUserStats(arg0 context.Context, arg1 *emptypb.Empty) (*cortexadmin.UserIDStatsList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AllUserStats", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.UserIDStatsList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AllUserStats indicates an expected call of AllUserStats. +func (mr *MockCortexAdminServerMockRecorder) AllUserStats(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllUserStats", reflect.TypeOf((*MockCortexAdminServer)(nil).AllUserStats), arg0, arg1) +} + +// DeleteRule mocks base method. +func (m *MockCortexAdminServer) DeleteRule(arg0 context.Context, arg1 *cortexadmin.DeleteRuleRequest) (*emptypb.Empty, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteRule", arg0, arg1) + ret0, _ := ret[0].(*emptypb.Empty) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteRule indicates an expected call of DeleteRule. +func (mr *MockCortexAdminServerMockRecorder) DeleteRule(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRule", reflect.TypeOf((*MockCortexAdminServer)(nil).DeleteRule), arg0, arg1) +} + +// ExtractRawSeries mocks base method. +func (m *MockCortexAdminServer) ExtractRawSeries(arg0 context.Context, arg1 *cortexadmin.MatcherRequest) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ExtractRawSeries", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ExtractRawSeries indicates an expected call of ExtractRawSeries. +func (mr *MockCortexAdminServerMockRecorder) ExtractRawSeries(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExtractRawSeries", reflect.TypeOf((*MockCortexAdminServer)(nil).ExtractRawSeries), arg0, arg1) +} + +// FlushBlocks mocks base method. +func (m *MockCortexAdminServer) FlushBlocks(arg0 context.Context, arg1 *emptypb.Empty) (*emptypb.Empty, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FlushBlocks", arg0, arg1) + ret0, _ := ret[0].(*emptypb.Empty) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FlushBlocks indicates an expected call of FlushBlocks. +func (mr *MockCortexAdminServerMockRecorder) FlushBlocks(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FlushBlocks", reflect.TypeOf((*MockCortexAdminServer)(nil).FlushBlocks), arg0, arg1) +} + +// GetCortexConfig mocks base method. +func (m *MockCortexAdminServer) GetCortexConfig(arg0 context.Context, arg1 *cortexadmin.ConfigRequest) (*cortexadmin.ConfigResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCortexConfig", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.ConfigResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCortexConfig indicates an expected call of GetCortexConfig. +func (mr *MockCortexAdminServerMockRecorder) GetCortexConfig(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCortexConfig", reflect.TypeOf((*MockCortexAdminServer)(nil).GetCortexConfig), arg0, arg1) +} + +// GetCortexStatus mocks base method. +func (m *MockCortexAdminServer) GetCortexStatus(arg0 context.Context, arg1 *emptypb.Empty) (*cortexadmin.CortexStatus, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCortexStatus", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.CortexStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCortexStatus indicates an expected call of GetCortexStatus. +func (mr *MockCortexAdminServerMockRecorder) GetCortexStatus(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCortexStatus", reflect.TypeOf((*MockCortexAdminServer)(nil).GetCortexStatus), arg0, arg1) +} + +// GetMetricLabelSets mocks base method. +func (m *MockCortexAdminServer) GetMetricLabelSets(arg0 context.Context, arg1 *cortexadmin.LabelRequest) (*cortexadmin.MetricLabels, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMetricLabelSets", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.MetricLabels) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMetricLabelSets indicates an expected call of GetMetricLabelSets. +func (mr *MockCortexAdminServerMockRecorder) GetMetricLabelSets(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetricLabelSets", reflect.TypeOf((*MockCortexAdminServer)(nil).GetMetricLabelSets), arg0, arg1) +} + +// GetMetricMetadata mocks base method. +func (m *MockCortexAdminServer) GetMetricMetadata(arg0 context.Context, arg1 *cortexadmin.MetricMetadataRequest) (*cortexadmin.MetricMetadata, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetMetricMetadata", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.MetricMetadata) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetMetricMetadata indicates an expected call of GetMetricMetadata. +func (mr *MockCortexAdminServerMockRecorder) GetMetricMetadata(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMetricMetadata", reflect.TypeOf((*MockCortexAdminServer)(nil).GetMetricMetadata), arg0, arg1) +} + +// GetRule mocks base method. +func (m *MockCortexAdminServer) GetRule(arg0 context.Context, arg1 *cortexadmin.GetRuleRequest) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRule", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRule indicates an expected call of GetRule. +func (mr *MockCortexAdminServerMockRecorder) GetRule(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRule", reflect.TypeOf((*MockCortexAdminServer)(nil).GetRule), arg0, arg1) +} + +// GetSeriesMetrics mocks base method. +func (m *MockCortexAdminServer) GetSeriesMetrics(arg0 context.Context, arg1 *cortexadmin.SeriesRequest) (*cortexadmin.SeriesInfoList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSeriesMetrics", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.SeriesInfoList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSeriesMetrics indicates an expected call of GetSeriesMetrics. +func (mr *MockCortexAdminServerMockRecorder) GetSeriesMetrics(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSeriesMetrics", reflect.TypeOf((*MockCortexAdminServer)(nil).GetSeriesMetrics), arg0, arg1) +} + +// ListRules mocks base method. +func (m *MockCortexAdminServer) ListRules(arg0 context.Context, arg1 *cortexadmin.ListRulesRequest) (*cortexadmin.ListRulesResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRules", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.ListRulesResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRules indicates an expected call of ListRules. +func (mr *MockCortexAdminServerMockRecorder) ListRules(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRules", reflect.TypeOf((*MockCortexAdminServer)(nil).ListRules), arg0, arg1) +} + +// LoadRules mocks base method. +func (m *MockCortexAdminServer) LoadRules(arg0 context.Context, arg1 *cortexadmin.LoadRuleRequest) (*emptypb.Empty, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadRules", arg0, arg1) + ret0, _ := ret[0].(*emptypb.Empty) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LoadRules indicates an expected call of LoadRules. +func (mr *MockCortexAdminServerMockRecorder) LoadRules(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadRules", reflect.TypeOf((*MockCortexAdminServer)(nil).LoadRules), arg0, arg1) +} + +// Query mocks base method. +func (m *MockCortexAdminServer) Query(arg0 context.Context, arg1 *cortexadmin.QueryRequest) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Query", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Query indicates an expected call of Query. +func (mr *MockCortexAdminServerMockRecorder) Query(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Query", reflect.TypeOf((*MockCortexAdminServer)(nil).Query), arg0, arg1) +} + +// QueryRange mocks base method. +func (m *MockCortexAdminServer) QueryRange(arg0 context.Context, arg1 *cortexadmin.QueryRangeRequest) (*cortexadmin.QueryResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "QueryRange", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.QueryResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QueryRange indicates an expected call of QueryRange. +func (mr *MockCortexAdminServerMockRecorder) QueryRange(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryRange", reflect.TypeOf((*MockCortexAdminServer)(nil).QueryRange), arg0, arg1) +} + +// WriteMetrics mocks base method. +func (m *MockCortexAdminServer) WriteMetrics(arg0 context.Context, arg1 *cortexadmin.WriteRequest) (*cortexadmin.WriteResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WriteMetrics", arg0, arg1) + ret0, _ := ret[0].(*cortexadmin.WriteResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// WriteMetrics indicates an expected call of WriteMetrics. +func (mr *MockCortexAdminServerMockRecorder) WriteMetrics(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteMetrics", reflect.TypeOf((*MockCortexAdminServer)(nil).WriteMetrics), arg0, arg1) +} + +// mustEmbedUnimplementedCortexAdminServer mocks base method. +func (m *MockCortexAdminServer) mustEmbedUnimplementedCortexAdminServer() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "mustEmbedUnimplementedCortexAdminServer") +} + +// mustEmbedUnimplementedCortexAdminServer indicates an expected call of mustEmbedUnimplementedCortexAdminServer. +func (mr *MockCortexAdminServerMockRecorder) mustEmbedUnimplementedCortexAdminServer() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "mustEmbedUnimplementedCortexAdminServer", reflect.TypeOf((*MockCortexAdminServer)(nil).mustEmbedUnimplementedCortexAdminServer)) +} + +// MockUnsafeCortexAdminServer is a mock of UnsafeCortexAdminServer interface. +type MockUnsafeCortexAdminServer struct { + ctrl *gomock.Controller + recorder *MockUnsafeCortexAdminServerMockRecorder +} + +// MockUnsafeCortexAdminServerMockRecorder is the mock recorder for MockUnsafeCortexAdminServer. +type MockUnsafeCortexAdminServerMockRecorder struct { + mock *MockUnsafeCortexAdminServer +} + +// NewMockUnsafeCortexAdminServer creates a new mock instance. +func NewMockUnsafeCortexAdminServer(ctrl *gomock.Controller) *MockUnsafeCortexAdminServer { + mock := &MockUnsafeCortexAdminServer{ctrl: ctrl} + mock.recorder = &MockUnsafeCortexAdminServerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockUnsafeCortexAdminServer) EXPECT() *MockUnsafeCortexAdminServerMockRecorder { + return m.recorder +} + +// mustEmbedUnimplementedCortexAdminServer mocks base method. +func (m *MockUnsafeCortexAdminServer) mustEmbedUnimplementedCortexAdminServer() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "mustEmbedUnimplementedCortexAdminServer") +} + +// mustEmbedUnimplementedCortexAdminServer indicates an expected call of mustEmbedUnimplementedCortexAdminServer. +func (mr *MockUnsafeCortexAdminServerMockRecorder) mustEmbedUnimplementedCortexAdminServer() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "mustEmbedUnimplementedCortexAdminServer", reflect.TypeOf((*MockUnsafeCortexAdminServer)(nil).mustEmbedUnimplementedCortexAdminServer)) +} diff --git a/pkg/test/mock/mockgen.yaml b/pkg/test/mock/mockgen.yaml index c5b32345e0..dbf8f37608 100644 --- a/pkg/test/mock/mockgen.yaml +++ b/pkg/test/mock/mockgen.yaml @@ -50,3 +50,16 @@ mocks: dest: pkg/test/mock/update/update.go types: - UpdateTypeHandler + - source: plugins/metrics/apis/cortexadmin/cortexadmin_grpc.pb.go + dest: pkg/test/mock/cortexadmin/cortexadmin.go + types: + - CortexAdminClient + - source: pkg/slo/backend/service.go + dest: pkg/test/mock/slo/service.go + types: + - ServiceBackend + - source: pkg/slo/backend/store.go + dest : pkg/test/mock/slo/store.go + types: + - SLOStore + diff --git a/pkg/test/mock/slo/service.go b/pkg/test/mock/slo/service.go new file mode 100644 index 0000000000..5c44d5d3f4 --- /dev/null +++ b/pkg/test/mock/slo/service.go @@ -0,0 +1,81 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: pkg/slo/backend/service.go + +// Package mock_backend is a generated GoMock package. +package mock_backend + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/rancher/opni/pkg/apis/slo/v1" +) + +// MockServiceBackend is a mock of ServiceBackend interface. +type MockServiceBackend struct { + ctrl *gomock.Controller + recorder *MockServiceBackendMockRecorder +} + +// MockServiceBackendMockRecorder is the mock recorder for MockServiceBackend. +type MockServiceBackendMockRecorder struct { + mock *MockServiceBackend +} + +// NewMockServiceBackend creates a new mock instance. +func NewMockServiceBackend(ctrl *gomock.Controller) *MockServiceBackend { + mock := &MockServiceBackend{ctrl: ctrl} + mock.recorder = &MockServiceBackendMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockServiceBackend) EXPECT() *MockServiceBackendMockRecorder { + return m.recorder +} + +// ListEvents mocks base method. +func (m *MockServiceBackend) ListEvents(ctx context.Context, req *v1.ListEventsRequest) (*v1.EventList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListEvents", ctx, req) + ret0, _ := ret[0].(*v1.EventList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListEvents indicates an expected call of ListEvents. +func (mr *MockServiceBackendMockRecorder) ListEvents(ctx, req interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListEvents", reflect.TypeOf((*MockServiceBackend)(nil).ListEvents), ctx, req) +} + +// ListMetrics mocks base method. +func (m *MockServiceBackend) ListMetrics(ctx context.Context, req *v1.ListMetricsRequest) (*v1.MetricGroupList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListMetrics", ctx, req) + ret0, _ := ret[0].(*v1.MetricGroupList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListMetrics indicates an expected call of ListMetrics. +func (mr *MockServiceBackendMockRecorder) ListMetrics(ctx, req interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMetrics", reflect.TypeOf((*MockServiceBackend)(nil).ListMetrics), ctx, req) +} + +// ListServices mocks base method. +func (m *MockServiceBackend) ListServices(ctx context.Context, req *v1.ListServicesRequest) (*v1.ServiceList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListServices", ctx, req) + ret0, _ := ret[0].(*v1.ServiceList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListServices indicates an expected call of ListServices. +func (mr *MockServiceBackendMockRecorder) ListServices(ctx, req interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListServices", reflect.TypeOf((*MockServiceBackend)(nil).ListServices), ctx, req) +} diff --git a/pkg/test/mock/slo/store.go b/pkg/test/mock/slo/store.go new file mode 100644 index 0000000000..757d924db7 --- /dev/null +++ b/pkg/test/mock/slo/store.go @@ -0,0 +1,143 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: pkg/slo/backend/store.go + +// Package mock_backend is a generated GoMock package. +package mock_backend + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/rancher/opni/pkg/apis/core/v1" + v10 "github.com/rancher/opni/pkg/apis/slo/v1" +) + +// MockSLOStore is a mock of SLOStore interface. +type MockSLOStore struct { + ctrl *gomock.Controller + recorder *MockSLOStoreMockRecorder +} + +// MockSLOStoreMockRecorder is the mock recorder for MockSLOStore. +type MockSLOStoreMockRecorder struct { + mock *MockSLOStore +} + +// NewMockSLOStore creates a new mock instance. +func NewMockSLOStore(ctrl *gomock.Controller) *MockSLOStore { + mock := &MockSLOStore{ctrl: ctrl} + mock.recorder = &MockSLOStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSLOStore) EXPECT() *MockSLOStoreMockRecorder { + return m.recorder +} + +// Clone mocks base method. +func (m *MockSLOStore) Clone(ctx context.Context, clone *v10.SLOData) (*v1.Reference, *v10.SLOData, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Clone", ctx, clone) + ret0, _ := ret[0].(*v1.Reference) + ret1, _ := ret[1].(*v10.SLOData) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// Clone indicates an expected call of Clone. +func (mr *MockSLOStoreMockRecorder) Clone(ctx, clone interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Clone", reflect.TypeOf((*MockSLOStore)(nil).Clone), ctx, clone) +} + +// Create mocks base method. +func (m *MockSLOStore) Create(ctx context.Context, req *v10.CreateSLORequest) (*v1.Reference, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", ctx, req) + ret0, _ := ret[0].(*v1.Reference) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Create indicates an expected call of Create. +func (mr *MockSLOStoreMockRecorder) Create(ctx, req interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockSLOStore)(nil).Create), ctx, req) +} + +// Delete mocks base method. +func (m *MockSLOStore) Delete(ctx context.Context, existing *v10.SLOData) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, existing) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockSLOStoreMockRecorder) Delete(ctx, existing interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockSLOStore)(nil).Delete), ctx, existing) +} + +// MultiClusterClone mocks base method. +func (m *MockSLOStore) MultiClusterClone(ctx context.Context, slo *v10.SLOData, toClusters []*v1.Reference) ([]*v1.Reference, []*v10.SLOData, []error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MultiClusterClone", ctx, slo, toClusters) + ret0, _ := ret[0].([]*v1.Reference) + ret1, _ := ret[1].([]*v10.SLOData) + ret2, _ := ret[2].([]error) + return ret0, ret1, ret2 +} + +// MultiClusterClone indicates an expected call of MultiClusterClone. +func (mr *MockSLOStoreMockRecorder) MultiClusterClone(ctx, slo, toClusters interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MultiClusterClone", reflect.TypeOf((*MockSLOStore)(nil).MultiClusterClone), ctx, slo, toClusters) +} + +// Preview mocks base method. +func (m *MockSLOStore) Preview(ctx context.Context, s *v10.CreateSLORequest) (*v10.SLOPreviewResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Preview", ctx, s) + ret0, _ := ret[0].(*v10.SLOPreviewResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Preview indicates an expected call of Preview. +func (mr *MockSLOStoreMockRecorder) Preview(ctx, s interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Preview", reflect.TypeOf((*MockSLOStore)(nil).Preview), ctx, s) +} + +// Status mocks base method. +func (m *MockSLOStore) Status(ctx context.Context, existing *v10.SLOData) (*v10.SLOStatus, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Status", ctx, existing) + ret0, _ := ret[0].(*v10.SLOStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Status indicates an expected call of Status. +func (mr *MockSLOStoreMockRecorder) Status(ctx, existing interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Status", reflect.TypeOf((*MockSLOStore)(nil).Status), ctx, existing) +} + +// Update mocks base method. +func (m *MockSLOStore) Update(ctx context.Context, incoming, existing *v10.SLOData) (*v10.SLOData, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", ctx, incoming, existing) + ret0, _ := ret[0].(*v10.SLOData) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Update indicates an expected call of Update. +func (mr *MockSLOStoreMockRecorder) Update(ctx, incoming, existing interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockSLOStore)(nil).Update), ctx, incoming, existing) +} diff --git a/plugins/alerting/pkg/alerting/plugin.go b/plugins/alerting/pkg/alerting/plugin.go index ca17f872df..07a7a7cf7d 100644 --- a/plugins/alerting/pkg/alerting/plugin.go +++ b/plugins/alerting/pkg/alerting/plugin.go @@ -10,6 +10,7 @@ import ( "github.com/rancher/opni/pkg/storage" "github.com/rancher/opni/plugins/alerting/apis/alertops" metricsExporter "github.com/rancher/opni/plugins/alerting/pkg/alerting/metrics" + "github.com/rancher/opni/plugins/alerting/pkg/alerting/slo/v1" "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" "github.com/rancher/opni/plugins/metrics/apis/cortexops" metricsdk "go.opentelemetry.io/otel/sdk/metric" @@ -31,6 +32,7 @@ import ( capabilityv1 "github.com/rancher/opni/pkg/apis/capability/v1" managementv1 "github.com/rancher/opni/pkg/apis/management/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" "github.com/rancher/opni/pkg/logger" managementext "github.com/rancher/opni/pkg/plugins/apis/apiextensions/management" streamext "github.com/rancher/opni/pkg/plugins/apis/apiextensions/stream" @@ -47,6 +49,7 @@ func (p *Plugin) Components() []server.ServerComponent { p.NotificationServerComponent, p.EndpointServerComponent, p.AlarmServerComponent, + p.SLOServerComponent, } } @@ -73,6 +76,7 @@ type Plugin struct { cortexOpsClient future.Future[cortexops.CortexOpsClient] natsConn future.Future[*nats.Conn] js future.Future[nats.JetStreamContext] + sloStorage future.Future[slo.StorageAPIs] globalWatchers management.ConditionWatcher gatewayConfig future.Future[*v1beta1.GatewayConfig] @@ -82,6 +86,7 @@ type Plugin struct { *notifications.NotificationServerComponent *endpoints.EndpointServerComponent *alarms.AlarmServerComponent + *slo.SLOServerComponent node node_backend.AlertingNodeBackend @@ -120,6 +125,7 @@ func NewPlugin(ctx context.Context) *Plugin { cortexOpsClient: future.New[cortexops.CortexOpsClient](), natsConn: future.New[*nats.Conn](), js: future.New[nats.JetStreamContext](), + sloStorage: future.New[slo.StorageAPIs](), gatewayConfig: future.New[*v1beta1.GatewayConfig](), @@ -156,6 +162,10 @@ func NewPlugin(ctx context.Context) *Plugin { p.NotificationServerComponent, ) + p.SLOServerComponent = slo.NewSLOServerComponent( + p.logger.With("component", "slo"), + ) + future.Wait4( p.storageBackend, p.mgmtClient, @@ -196,6 +206,16 @@ func NewPlugin(ctx context.Context) *Plugin { ) }) + future.Wait2(p.mgmtClient, p.sloStorage, func( + mgmtClient managementv1.ManagementClient, + sloStorage slo.StorageAPIs, + ) { + p.SLOServerComponent.Initialize(slo.SLOServerConfiguration{ + MgmtClient: mgmtClient, + Storage: sloStorage, + }) + }) + future.Wait5(p.js, p.storageClientSet, p.mgmtClient, p.adminClient, p.cortexOpsClient, func( js nats.JetStreamContext, @@ -267,6 +287,10 @@ func Scheme(ctx context.Context) meta.Scheme { &node.NodeAlertingCapability_ServiceDesc, &p.node, ), + util.PackService( + &slov1.SLO_ServiceDesc, + p.SLOServerComponent, + ), ), ) diff --git a/plugins/alerting/pkg/alerting/slo/v1/component.go b/plugins/alerting/pkg/alerting/slo/v1/component.go new file mode 100644 index 0000000000..9d29d23626 --- /dev/null +++ b/plugins/alerting/pkg/alerting/slo/v1/component.go @@ -0,0 +1,96 @@ +package slo + +import ( + "context" + "sync" + + "github.com/rancher/opni/pkg/slo/backend" + "github.com/rancher/opni/pkg/storage" + + "github.com/rancher/opni/pkg/alerting/server" + alertingSync "github.com/rancher/opni/pkg/alerting/server/sync" + managementv1 "github.com/rancher/opni/pkg/apis/management/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/util" + "github.com/rancher/opni/pkg/util/future" + "go.uber.org/zap" +) + +type SLOServerComponent struct { + slov1.UnsafeSLOServer + util.Initializer + logger *zap.SugaredLogger + storage future.Future[StorageAPIs] + mgmtClient future.Future[managementv1.ManagementClient] +} + +var _ server.ServerComponent = (*SLOServerComponent)(nil) +var _ slov1.SLOServer = (*SLOServerComponent)(nil) + +type StorageAPIs struct { + SLOs storage.KeyValueStoreT[*slov1.SLOData] + Services storage.KeyValueStoreT[*slov1.Service] + Metrics storage.KeyValueStoreT[*slov1.Metric] +} + +func NewSLOServerComponent( + logger *zap.SugaredLogger, +) *SLOServerComponent { + return &SLOServerComponent{ + logger: logger, + storage: future.New[StorageAPIs](), + mgmtClient: future.New[managementv1.ManagementClient](), + } +} + +type SLOServerConfiguration struct { + Storage StorageAPIs + MgmtClient managementv1.ManagementClient +} + +func (s *SLOServerComponent) Name() string { + return "notification" +} + +func (s *SLOServerComponent) Status() server.Status { + return server.Status{ + Running: s.Initialized(), + } +} + +func (s *SLOServerComponent) Ready() bool { + return s.Initialized() +} + +func (s *SLOServerComponent) Healthy() bool { + return s.Initialized() +} + +func (s *SLOServerComponent) SetConfig(_ server.Config) { +} + +func (s *SLOServerComponent) Sync(_ context.Context, _ alertingSync.SyncInfo) error { + return nil +} + +func (s *SLOServerComponent) Initialize(conf SLOServerConfiguration) { + s.InitOnce(func() { + s.mgmtClient.Set(conf.MgmtClient) + s.storage.Set(conf.Storage) + }) +} + +var datasourceMu sync.RWMutex + +var ( + datasources = make(map[string]backend.SLODatasource) +) + +func RegisterDatasource( + datasourceName string, + datasourceImpl backend.SLODatasource, +) { + defer datasourceMu.Unlock() + datasourceMu.Lock() + datasources[datasourceName] = datasourceImpl +} diff --git a/plugins/alerting/pkg/alerting/slo/v1/server.go b/plugins/alerting/pkg/alerting/slo/v1/server.go new file mode 100644 index 0000000000..50f1353b13 --- /dev/null +++ b/plugins/alerting/pkg/alerting/slo/v1/server.go @@ -0,0 +1,311 @@ +package slo + +import ( + "context" + "fmt" + "path" + "time" + + corev1 "github.com/rancher/opni/pkg/apis/core/v1" + managementv1 "github.com/rancher/opni/pkg/apis/management/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/slo/shared" + "github.com/rancher/opni/pkg/storage" + "github.com/rancher/opni/pkg/validation" + "github.com/samber/lo" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func list[T proto.Message](ctx context.Context, kvc storage.KeyValueStoreT[T], prefix string) ([]T, error) { + keys, err := kvc.ListKeys(ctx, prefix) + if err != nil { + return nil, err + } + items := make([]T, len(keys)) + for i, key := range keys { + item, err := kvc.Get(ctx, key) + if err != nil { + return nil, err + } + items[i] = item + } + return items, nil +} + +func checkDatasource(datasource string) error { + datasourceMu.RLock() + defer datasourceMu.RUnlock() + if _, ok := datasources[datasource]; !ok { + return shared.ErrInvalidDatasource + } + return nil +} + +func (s *SLOServerComponent) GetSLO(ctx context.Context, ref *corev1.Reference) (*slov1.SLOData, error) { + return s.storage.Get().SLOs.Get(ctx, path.Join("/slos", ref.Id)) +} + +func (s *SLOServerComponent) ListSLOs(ctx context.Context, _ *emptypb.Empty) (*slov1.ServiceLevelObjectiveList, error) { + items, err := list(ctx, s.storage.Get().SLOs, "/slos") + if err != nil { + return nil, err + } + return &slov1.ServiceLevelObjectiveList{ + Items: items, + }, nil +} + +func (s *SLOServerComponent) CreateSLO(ctx context.Context, req *slov1.CreateSLORequest) (*corev1.Reference, error) { + if err := req.Validate(); err != nil { + return nil, err + } + clusterList, err := s.mgmtClient.Get().ListClusters(ctx, &managementv1.ListClustersRequest{}) + if err != nil { + return nil, err + } + if !clusterList.Has(req.Slo.ClusterId) { + return nil, validation.Error("invalid cluster : not found") + } + datasourceMu.RLock() + sloStore := datasources[req.GetSlo().GetDatasource()] + datasourceMu.RUnlock() + + if err := sloStore.Precondition(ctx, &corev1.Reference{Id: req.Slo.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + + id, err := sloStore.Create(ctx, req) + if err != nil { + return nil, err + } + sloData := &slov1.SLOData{ + Id: id.Id, + SLO: req.GetSlo(), + CreatedAt: timestamppb.New(time.Now()), + } + if err := s.storage.Get().SLOs.Put(ctx, path.Join("/slos", id.Id), sloData); err != nil { + return nil, err + } + return id, nil +} + +func (s *SLOServerComponent) UpdateSLO(ctx context.Context, req *slov1.SLOData) (*emptypb.Empty, error) { + if err := req.Validate(); err != nil { + return nil, err + } + clusterList, err := s.mgmtClient.Get().ListClusters(ctx, &managementv1.ListClustersRequest{}) + if err != nil { + return nil, err + } + validCluster := false + for _, clusterListItem := range clusterList.Items { + if clusterListItem.Id == req.SLO.ClusterId { + validCluster = true + break + } + } + if !validCluster { + return nil, validation.Error("invalid cluster") + } + existing, err := s.storage.Get().SLOs.Get(ctx, path.Join("/slos", req.Id)) + if err != nil { + return nil, err + } + sloStore := datasources[req.GetSLO().GetDatasource()] + if err := sloStore.Precondition(ctx, &corev1.Reference{Id: req.SLO.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + updatedSLO, err := sloStore.Update(ctx, req, existing) + if err != nil { // exit when update fails + return nil, err + } + // Merge when everything else is done + if err := s.storage.Get().SLOs.Put(ctx, path.Join("/slos", req.Id), updatedSLO); err != nil { + return nil, err + } + return &emptypb.Empty{}, nil +} + +func (s *SLOServerComponent) DeleteSLO(ctx context.Context, req *corev1.Reference) (*emptypb.Empty, error) { + lg := s.logger + existing, err := s.storage.Get().SLOs.Get(ctx, path.Join("/slos", req.Id)) + if err != nil { + lg.With("delete slo", req.Id).Error("failed to get slo to delete in K,V store") + return nil, err + } + if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { + return nil, err + } + sloStore := datasources[existing.SLO.GetDatasource()] + if err := sloStore.Precondition(ctx, &corev1.Reference{Id: existing.SLO.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + err = sloStore.Delete(ctx, existing) + if err != nil { + return nil, err + } + if err := s.storage.Get().SLOs.Delete(ctx, path.Join("/slos", req.Id)); err != nil { + return nil, err + } + return &emptypb.Empty{}, nil +} + +func (s *SLOServerComponent) CloneSLO(ctx context.Context, ref *corev1.Reference) (*slov1.SLOData, error) { + existing, err := s.storage.Get().SLOs.Get(ctx, path.Join("/slos", ref.Id)) + if err != nil { + return nil, err + } + if err := existing.Validate(); err != nil { + return nil, err + } + if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { + return nil, err + } + sloStore := datasources[existing.SLO.GetDatasource()] + if err := sloStore.Precondition(ctx, &corev1.Reference{Id: existing.SLO.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + newId, newData, anyError := sloStore.Clone(ctx, existing) + newData.CreatedAt = timestamppb.New(time.Now()) + if anyError != nil { + return nil, anyError + } + if err := s.storage.Get().SLOs.Put(ctx, path.Join("/slos", newId.Id), newData); err != nil { + return nil, err + } + return newData, nil +} + +func (s *SLOServerComponent) CloneToClusters(ctx context.Context, req *slov1.MultiClusterSLO) (*slov1.MultiClusterFailures, error) { + existing, err := s.storage.Get().SLOs.Get(ctx, path.Join("/slos", req.CloneId.Id)) + if err != nil { + return nil, err + } + if err := existing.Validate(); err != nil { + return nil, err + } + if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { + return nil, err + } + sloStore := datasources[existing.SLO.GetDatasource()] + for _, cluster := range req.Clusters { + if err := sloStore.Precondition(ctx, cluster); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + } + failures := []string{} + createdIds, createdData, errors := sloStore.MultiClusterClone(ctx, existing, req.Clusters) + createdAt := timestamppb.New(time.Now()) + for idx, err := range errors { + if err != nil { + failures = append(failures, err.Error()) + } else { + createdData[idx].CreatedAt = createdAt + if err := s.storage.Get().SLOs.Put( + ctx, + path.Join("/slos", createdIds[idx].Id), + createdData[idx], + ); err != nil { + failures = append(failures, fmt.Sprintf("failed to persist SLO configuration : %s", err.Error())) + } + } + } + return &slov1.MultiClusterFailures{ + Failures: failures, + }, nil +} + +func (s *SLOServerComponent) GetSLOStatus(ctx context.Context, ref *corev1.Reference) (*slov1.SLOStatus, error) { + existing, err := s.storage.Get().SLOs.Get(ctx, path.Join("/slos", ref.Id)) + if err != nil { + return nil, err + } + if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { + return nil, err + } + sloStore := datasources[existing.SLO.GetDatasource()] + if err := sloStore.Precondition(ctx, &corev1.Reference{Id: existing.SLO.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + status, err := sloStore.Status(ctx, existing) + return status, err +} + +func (s *SLOServerComponent) Preview(ctx context.Context, req *slov1.CreateSLORequest) (*slov1.SLOPreviewResponse, error) { + if err := req.Validate(); err != nil { + return nil, err + } + if err := checkDatasource(req.GetSlo().GetDatasource()); err != nil { + return nil, err + } + sloStore := datasources[req.GetSlo().GetDatasource()] + if err := sloStore.Precondition(ctx, &corev1.Reference{Id: req.Slo.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + preview, err := sloStore.Preview(ctx, req) + return preview, err +} + +// -------- Service Discovery --------- + +func (s *SLOServerComponent) ListServices(ctx context.Context, req *slov1.ListServicesRequest) (*slov1.ServiceList, error) { + err := checkDatasource(req.Datasource) + if err != nil { + return nil, shared.ErrInvalidDatasource + } + datasourceMu.RLock() + backend := datasources[req.Datasource] + datasourceMu.RUnlock() + + if err := backend.Precondition(ctx, &corev1.Reference{Id: req.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + return backend.ListServices(ctx, req) +} + +func (s *SLOServerComponent) ListMetrics(ctx context.Context, req *slov1.ListMetricsRequest) (*slov1.MetricGroupList, error) { + err := checkDatasource(req.Datasource) + if err != nil { + return nil, shared.ErrInvalidDatasource + } + datasourceMu.RLock() + backend := datasources[req.Datasource] + datasourceMu.RUnlock() + + if err := backend.Precondition(ctx, &corev1.Reference{Id: req.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + return backend.ListMetrics(ctx, req) +} + +func (s *SLOServerComponent) ListEvents(ctx context.Context, req *slov1.ListEventsRequest) (*slov1.EventList, error) { + // fetch labels & their label values for the given cluster & service + if err := req.Validate(); err != nil { + return nil, err + } + datasource := req.GetDatasource() + if err := checkDatasource(datasource); err != nil { + return nil, shared.ErrInvalidDatasource + } + datasourceMu.RLock() + backend := datasources[req.Datasource] + datasourceMu.RUnlock() + + if err := backend.Precondition(ctx, &corev1.Reference{Id: req.ClusterId}); err != nil { + return nil, status.Error(codes.FailedPrecondition, err.Error()) + } + return backend.ListEvents(ctx, req) +} + +func (s *SLOServerComponent) ListDatasources(_ context.Context, _ *emptypb.Empty) (*slov1.DatasourceInfo, error) { + datasourceMu.RLock() + datasources := lo.Keys(datasources) + defer datasourceMu.RUnlock() + return &slov1.DatasourceInfo{ + Items: datasources, + }, nil +} diff --git a/plugins/alerting/pkg/alerting/system.go b/plugins/alerting/pkg/alerting/system.go index e37dd3158d..23888a0a94 100644 --- a/plugins/alerting/pkg/alerting/system.go +++ b/plugins/alerting/pkg/alerting/system.go @@ -10,10 +10,15 @@ import ( "time" "github.com/rancher/opni/pkg/caching" + "github.com/rancher/opni/pkg/capabilities" + "github.com/rancher/opni/pkg/capabilities/wellknown" "github.com/rancher/opni/pkg/management" + "github.com/rancher/opni/pkg/slo/backend" + "github.com/rancher/opni/pkg/slo/backend/metrics" "github.com/rancher/opni/pkg/storage" "github.com/rancher/opni/plugins/alerting/apis/alertops" "github.com/rancher/opni/plugins/alerting/pkg/alerting/alarms/v1" + "github.com/rancher/opni/plugins/alerting/pkg/alerting/slo/v1" "github.com/rancher/opni/plugins/alerting/pkg/node_backend" "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" "github.com/rancher/opni/plugins/metrics/apis/cortexops" @@ -26,6 +31,7 @@ import ( "github.com/rancher/opni/pkg/alerting/server" capabilityv1 "github.com/rancher/opni/pkg/apis/capability/v1" + corev1 "github.com/rancher/opni/pkg/apis/core/v1" managementv1 "github.com/rancher/opni/pkg/apis/management/v1" "github.com/rancher/opni/pkg/config/v1beta1" "github.com/rancher/opni/pkg/machinery" @@ -37,6 +43,7 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/emptypb" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" _ "github.com/rancher/opni/pkg/storage/etcd" _ "github.com/rancher/opni/pkg/storage/jetstream" ) @@ -96,6 +103,12 @@ func (p *Plugin) UseKeyValueStore(client system.KeyValueStoreClient) { NodeCapabilitySpecs: storage.NewKeyValueStoreWithPrefix(system.NewKVStoreClient[*node.AlertingCapabilitySpec](client), "/alerting/config/capability/nodes"), }) + p.sloStorage.Set(slo.StorageAPIs{ + SLOs: system.NewKVStoreClient[*slov1.SLOData](client), + Services: system.NewKVStoreClient[*slov1.Service](client), + Metrics: system.NewKVStoreClient[*slov1.Metric](client), + }) + var ( nc *nats.Conn err error @@ -193,6 +206,8 @@ func (p *Plugin) UseAPIExtensions(intf system.ExtensionClientInterface) { } p.adminClient.Set(cortexadmin.NewCortexAdminClient(cc)) p.cortexOpsClient.Set(cortexops.NewCortexOpsClient(cc)) + + p.registerMetricsDatasource() } func (p *Plugin) UseNodeManagerClient(client capabilityv1.NodeManagerClient) { @@ -200,6 +215,42 @@ func (p *Plugin) UseNodeManagerClient(client capabilityv1.NodeManagerClient) { <-p.ctx.Done() } +func (p *Plugin) registerMetricsDatasource() { + metricsProvider := metrics.NewProvider(p.logger.With("component", "slo")) + metricsProvider.Initialize(p.adminClient.Get()) + metricsStoreProvider := metrics.NewMetricsSLOStoreProvider(p.logger.With("component", "slo")) + metricsStoreProvider.Initialize(p.adminClient.Get()) + + metricsDatasource := backend.NewSLODatasource( + metrics.NewMetricsSLOStore( + metricsStoreProvider, + ), + metrics.NewBackend( + metricsProvider, + ), + func(ctx context.Context, clusterId *corev1.Reference) error { + mgmtClient, err := p.mgmtClient.GetContext(ctx) + if err != nil { + return err + } + cluster, err := mgmtClient.GetCluster(ctx, &corev1.Reference{ + Id: clusterId.Id, + }) + if err != nil { + return err + } + if !capabilities.Has(cluster, capabilities.Cluster(wellknown.CapabilityMetrics)) { + return fmt.Errorf("requires metrics capability to be installed on cluster %s", cluster.GetId()) + } + return nil + }, + ) + slo.RegisterDatasource( + "monitoring", + metricsDatasource, + ) +} + func (p *Plugin) handleDriverNotifications() { for { select { diff --git a/plugins/slo/apis/slo/compat.go b/plugins/slo/apis/slo/compat.go deleted file mode 100644 index f0da933ad3..0000000000 --- a/plugins/slo/apis/slo/compat.go +++ /dev/null @@ -1,11 +0,0 @@ -package slo - -import ( - "github.com/kralicky/ragu/compat" - _ "k8s.io/api/core/v1" -) - -func init() { - compat.LoadGogoFileDescriptor("k8s.io/api/core/v1/generated.proto") - compat.LoadGogoFileDescriptor("k8s.io/apimachinery/pkg/api/resource/generated.proto") -} diff --git a/plugins/slo/main.go b/plugins/slo/main.go deleted file mode 100644 index 4c572fbcb3..0000000000 --- a/plugins/slo/main.go +++ /dev/null @@ -1,16 +0,0 @@ -package main - -import ( - "github.com/rancher/opni/pkg/plugins" - "github.com/rancher/opni/pkg/plugins/meta" - "github.com/rancher/opni/plugins/slo/pkg/slo" -) - -func main() { - m := plugins.Main{ - Modes: meta.ModeSet{ - meta.ModeGateway: slo.Scheme, - }, - } - m.Exec() -} diff --git a/plugins/slo/pkg/slo/api.go b/plugins/slo/pkg/slo/api.go deleted file mode 100644 index 915c3bc675..0000000000 --- a/plugins/slo/pkg/slo/api.go +++ /dev/null @@ -1,266 +0,0 @@ -// API implementation - -package slo - -import ( - "context" - "path" - "time" - - corev1 "github.com/rancher/opni/pkg/apis/core/v1" - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - "github.com/rancher/opni/pkg/slo/shared" - "github.com/rancher/opni/pkg/storage" - "github.com/rancher/opni/pkg/validation" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/emptypb" - "google.golang.org/protobuf/types/known/timestamppb" -) - -func list[T proto.Message](ctx context.Context, kvc storage.KeyValueStoreT[T], prefix string) ([]T, error) { - keys, err := kvc.ListKeys(ctx, prefix) - if err != nil { - return nil, err - } - items := make([]T, len(keys)) - for i, key := range keys { - item, err := kvc.Get(ctx, key) - if err != nil { - return nil, err - } - items[i] = item - } - return items, nil -} - -func checkDatasource(datasource string) error { - if _, ok := datasourceToSLO[datasource]; !ok { - return shared.ErrInvalidDatasource - } - if _, ok := datasourceToService[datasource]; !ok { - return shared.ErrInvalidDatasource - } - return nil -} - -func (p *Plugin) GetSLO(ctx context.Context, ref *corev1.Reference) (*sloapi.SLOData, error) { - return p.storage.Get().SLOs.Get(ctx, path.Join("/slos", ref.Id)) -} - -func (p *Plugin) ListSLOs(ctx context.Context, _ *emptypb.Empty) (*sloapi.ServiceLevelObjectiveList, error) { - items, err := list(ctx, p.storage.Get().SLOs, "/slos") - if err != nil { - return nil, err - } - return &sloapi.ServiceLevelObjectiveList{ - Items: items, - }, nil -} - -func (p *Plugin) CreateSLO(ctx context.Context, slorequest *sloapi.CreateSLORequest) (*corev1.Reference, error) { - if err := slorequest.Validate(); err != nil { - return nil, err - } - clusterList, err := p.mgmtClient.Get().ListClusters(ctx, &managementv1.ListClustersRequest{}) - if err != nil { - return nil, err - } - validCluster := false - for _, clusterListItem := range clusterList.Items { - if clusterListItem.Id == slorequest.Slo.ClusterId { - validCluster = true - break - } - } - if !validCluster { - return nil, validation.Error("invalid cluster") - } - sloStore := datasourceToSLO[slorequest.GetSlo().GetDatasource()].WithCurrentRequest(ctx, slorequest) - id, err := sloStore.Create() - if err != nil { - return nil, err - } - sloData := &sloapi.SLOData{ - Id: id.Id, - SLO: slorequest.GetSlo(), - CreatedAt: timestamppb.New(time.Now()), - } - if err := p.storage.Get().SLOs.Put(ctx, path.Join("/slos", id.Id), sloData); err != nil { - return nil, err - } - return id, nil -} - -func (p *Plugin) UpdateSLO(ctx context.Context, req *sloapi.SLOData) (*emptypb.Empty, error) { - if err := req.Validate(); err != nil { - return nil, err - } - clusterList, err := p.mgmtClient.Get().ListClusters(ctx, &managementv1.ListClustersRequest{}) - if err != nil { - return nil, err - } - validCluster := false - for _, clusterListItem := range clusterList.Items { - if clusterListItem.Id == req.SLO.ClusterId { - validCluster = true - break - } - } - if !validCluster { - return nil, validation.Error("invalid cluster") - } - existing, err := p.storage.Get().SLOs.Get(ctx, path.Join("/slos", req.Id)) - if err != nil { - return nil, err - } - sloStore := datasourceToSLO[req.GetSLO().GetDatasource()].WithCurrentRequest(ctx, req) - updatedSLO, err := sloStore.Update(existing) - if err != nil { // exit when update fails - return nil, err - } - // Merge when everything else is done - if err := p.storage.Get().SLOs.Put(ctx, path.Join("/slos", req.Id), updatedSLO); err != nil { - return nil, err - } - return &emptypb.Empty{}, nil -} - -func (p *Plugin) DeleteSLO(ctx context.Context, req *corev1.Reference) (*emptypb.Empty, error) { - lg := p.logger - existing, err := p.storage.Get().SLOs.Get(ctx, path.Join("/slos", req.Id)) - if err != nil { - lg.With("delete slo", req.Id).Error("failed to get slo to delete in K,V store") - return nil, err - } - if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { - return nil, err - } - sloStore := datasourceToSLO[existing.SLO.GetDatasource()].WithCurrentRequest(ctx, req) - err = sloStore.Delete(existing) - if err != nil { - return nil, err - } - if err := p.storage.Get().SLOs.Delete(ctx, path.Join("/slos", req.Id)); err != nil { - return nil, err - } - return &emptypb.Empty{}, nil -} - -func (p *Plugin) CloneSLO(ctx context.Context, ref *corev1.Reference) (*sloapi.SLOData, error) { - existing, err := p.storage.Get().SLOs.Get(ctx, path.Join("/slos", ref.Id)) - if err != nil { - return nil, err - } - if err := existing.Validate(); err != nil { - return nil, err - } - if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { - return nil, err - } - sloStore := datasourceToSLO[existing.SLO.GetDatasource()].WithCurrentRequest(ctx, ref) - newId, newData, anyError := sloStore.Clone(existing) - newData.CreatedAt = timestamppb.New(time.Now()) - if anyError != nil { - return nil, anyError - } - if err := p.storage.Get().SLOs.Put(ctx, path.Join("/slos", newId.Id), newData); err != nil { - return nil, err - } - return newData, nil -} - -func (p *Plugin) CloneToClusters(ctx context.Context, req *sloapi.MultiClusterSLO) (*sloapi.MultiClusterFailures, error) { - existing, err := p.storage.Get().SLOs.Get(ctx, path.Join("/slos", req.CloneId.Id)) - if err != nil { - return nil, err - } - if err := existing.Validate(); err != nil { - return nil, err - } - if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { - return nil, err - } - sloStore := datasourceToSLO[existing.SLO.GetDatasource()].WithCurrentRequest(ctx, req) - svcBackend := datasourceToService[existing.SLO.GetDatasource()] // with current request is set in multi cluster clone - failures := []string{} - createdIds, createdData, errors := sloStore.MultiClusterClone(existing, req.Clusters, svcBackend) - createdAt := timestamppb.New(time.Now()) - for idx, err := range errors { - if err != nil { - failures = append(failures, err.Error()) - } else { - createdData[idx].CreatedAt = createdAt - if err := p.storage.Get().SLOs.Put( - ctx, - path.Join("/slos", createdIds[idx].Id), - createdData[idx], - ); err != nil { - //FIXME: maybe don't do this - return nil, err - } - } - } - return &sloapi.MultiClusterFailures{ - Failures: failures, - }, nil -} - -func (p *Plugin) Status(ctx context.Context, ref *corev1.Reference) (*sloapi.SLOStatus, error) { - existing, err := p.storage.Get().SLOs.Get(ctx, path.Join("/slos", ref.Id)) - if err != nil { - return nil, err - } - if err := checkDatasource(existing.SLO.GetDatasource()); err != nil { - return nil, err - } - sloStore := datasourceToSLO[existing.SLO.GetDatasource()].WithCurrentRequest(ctx, ref) - status, err := sloStore.Status(existing) - return status, err -} - -func (p *Plugin) Preview(ctx context.Context, req *sloapi.CreateSLORequest) (*sloapi.SLOPreviewResponse, error) { - if err := req.Validate(); err != nil { - return nil, err - } - slo := CreateSLORequestToStruct(req) - if err := checkDatasource(req.GetSlo().GetDatasource()); err != nil { - return nil, err - } - sloStore := datasourceToSLO[req.GetSlo().GetDatasource()].WithCurrentRequest(ctx, req) - return sloStore.Preview(slo) -} - -// -------- Service Discovery --------- - -func (p *Plugin) ListServices(ctx context.Context, req *sloapi.ListServicesRequest) (*sloapi.ServiceList, error) { - //lg := p.logger - err := checkDatasource(req.Datasource) - if err != nil { - return nil, shared.ErrInvalidDatasource - } - backend := datasourceToService[req.Datasource].WithCurrentRequest(ctx, req) - return backend.ListServices() -} - -func (p *Plugin) ListMetrics(ctx context.Context, req *sloapi.ListMetricsRequest) (*sloapi.MetricGroupList, error) { - err := checkDatasource(req.Datasource) - if err != nil { - return nil, shared.ErrInvalidDatasource - } - backend := datasourceToService[req.Datasource].WithCurrentRequest(ctx, req) - return backend.ListMetrics() -} - -func (p *Plugin) ListEvents(ctx context.Context, req *sloapi.ListEventsRequest) (*sloapi.EventList, error) { - // fetch labels & their label values for the given cluster & service - if err := req.Validate(); err != nil { - return nil, err - } - datasource := req.GetDatasource() - if err := checkDatasource(datasource); err != nil { - return nil, shared.ErrInvalidDatasource - } - backend := datasourceToService[datasource].WithCurrentRequest(ctx, req) - return backend.ListEvents() -} diff --git a/plugins/slo/pkg/slo/cortex_info.go b/plugins/slo/pkg/slo/cortex_info.go deleted file mode 100644 index 6878d34ab7..0000000000 --- a/plugins/slo/pkg/slo/cortex_info.go +++ /dev/null @@ -1,224 +0,0 @@ -package slo - -import ( - "context" - "fmt" - "time" - - "emperror.dev/errors" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/rulefmt" - "github.com/rancher/opni/pkg/metrics/compat" - "go.uber.org/zap" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/durationpb" - "google.golang.org/protobuf/types/known/timestamppb" - "gopkg.in/yaml.v3" -) - -var instantMaskDisabled = true - -func createGrafanaSLOMask(ctx context.Context, p *Plugin, clusterId string, ruleId string) error { - p.logger.With("sloId", ruleId, "clusterId", clusterId).Debugf("creating grafana mask") - if !instantMaskDisabled { - _, err := p.adminClient.Get().WriteMetrics(ctx, &cortexadmin.WriteRequest{ - ClusterID: clusterId, - Timeseries: []*cortexadmin.TimeSeries{ - { - Labels: []*cortexadmin.Label{ - { - Name: "__name__", //unique identifier for metrics - Value: grafana_delete_mask, - }, - { - Name: slo_uuid, - Value: ruleId, - }, - }, - }, - }, - }) - return err - } - return nil -} - -func tryApplyThenDeleteCortexRules( - ctx context.Context, - p *Plugin, - lg *zap.SugaredLogger, - clusterId string, - ruleId *string, - toApply []rulefmt.RuleGroup, -) error { - var errArr []error - for _, rules := range toApply { - err := applyCortexSLORules( - ctx, - p, - lg, - clusterId, - rules, - ) - if err != nil { - errArr = append(errArr, err) - } - } - if len(errArr) > 0 { - for _, rules := range toApply { - err := deleteCortexSLORules( - ctx, - p, - lg, - clusterId, - rules.Name, - ) - if err != nil { - errArr = append(errArr, err) - } - } - } - if ruleId != nil { - err := createGrafanaSLOMask(ctx, p, clusterId, *ruleId) - if err != nil { - lg.Errorf("creating grafana mask failed %s", err) - errArr = append(errArr, err) - } - } - - return errors.Combine(errArr...) -} - -// Apply Cortex Rules to Cortex separately : -// - recording rules -// - metadata rules -// - alert rules -func applyCortexSLORules( - ctx context.Context, - p *Plugin, - lg *zap.SugaredLogger, - clusterId string, - ruleSpec rulefmt.RuleGroup, -) error { - out, err := yaml.Marshal(ruleSpec) - if err != nil { - return err - } - - _, err = p.adminClient.Get().LoadRules(ctx, &cortexadmin.LoadRuleRequest{ - Namespace: "slo", - YamlContent: out, - ClusterId: clusterId, - }) - if err != nil { - lg.Error(fmt.Sprintf( - "Failed to load rules for cluster %s, rule : %s,", - clusterId, string(out))) - } - return err -} - -// } -func deleteCortexSLORules( - ctx context.Context, - p *Plugin, - _ *zap.SugaredLogger, - clusterId string, - groupName string, -) error { - _, err := p.adminClient.Get().DeleteRule(ctx, &cortexadmin.DeleteRuleRequest{ - ClusterId: clusterId, - Namespace: "slo", - GroupName: groupName, - }) - // we can ignore 404s here since if we can't find them, - // then it will be impossible to delete them anyway - if status.Code(err) == codes.NotFound || status.Code(err) == codes.OK { - return nil - } - return err -} - -func QuerySLOComponentByRecordName( - ctx context.Context, - client cortexadmin.CortexAdminClient, - recordName string, - clusterId string, -) (*model.Vector, error) { - resp, err := client.Query(ctx, &cortexadmin.QueryRequest{ - Tenants: []string{clusterId}, - Query: recordName, - }) - if err != nil { - return nil, err - } - rawBytes := resp.Data - qres, err := compat.UnmarshalPrometheusResponse(rawBytes) - if err != nil { - return nil, err - } - dataVector, err := qres.GetVector() - if err != nil { - return nil, err - } - return dataVector, nil -} - -func QuerySLOComponentByRawQuery( - ctx context.Context, - client cortexadmin.CortexAdminClient, - rawQuery string, - clusterId string, -) (*model.Vector, error) { - resp, err := client.Query(ctx, &cortexadmin.QueryRequest{ - Tenants: []string{clusterId}, - Query: rawQuery, - }) - if err != nil { - return nil, err - } - rawBytes := resp.Data - qres, err := compat.UnmarshalPrometheusResponse(rawBytes) - if err != nil { - return nil, err - } - dataVector, err := qres.GetVector() - if err != nil { - return nil, err - } - return dataVector, nil -} - -func QuerySLOComponentByRawQueryRange( - ctx context.Context, - client cortexadmin.CortexAdminClient, - rawQuery string, - clusterId string, - start time.Time, - end time.Time, - step time.Duration, -) (*model.Matrix, error) { - resp, err := client.QueryRange(ctx, &cortexadmin.QueryRangeRequest{ - Tenants: []string{clusterId}, - Query: rawQuery, - Start: timestamppb.New(start), - End: timestamppb.New(end), - Step: durationpb.New(step), - }) - if err != nil { - return nil, err - } - rawBytes := resp.Data - qres, err := compat.UnmarshalPrometheusResponse(rawBytes) - if err != nil { - return nil, err - } - dataMatrix, err := qres.GetMatrix() - if err != nil { - return nil, err - } - return dataMatrix, nil -} diff --git a/plugins/slo/pkg/slo/filters.go b/plugins/slo/pkg/slo/filters.go deleted file mode 100644 index 9b1706622f..0000000000 --- a/plugins/slo/pkg/slo/filters.go +++ /dev/null @@ -1,165 +0,0 @@ -package slo - -import ( - "embed" - "fmt" - "io/fs" - "path/filepath" - "regexp" - - "github.com/rancher/opni/pkg/logger" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "go.uber.org/zap" - "gopkg.in/yaml.v3" -) - -//go:embed metricgroups/*.yaml -var MetricGroups embed.FS - -//go:embed servicegroups/*.yaml -var ServiceGroups embed.FS - -// map of directory names to their embed.FS -var EnabledFilters = map[string]embed.FS{"metricgroups": MetricGroups, "servicegroups": ServiceGroups} -var filters = constructFilters(logger.NewPluginLogger().Named("slo")) - -// Regexp adds unmarshalling from json for regexp.Regexp -type Regexp struct { - *regexp.Regexp -} - -// UnmarshalText unmarshals json into a regexp.Regexp -func (r *Regexp) UnmarshalText(b []byte) error { - regex, err := regexp.Compile(string(b)) - if err != nil { - return err - } - - r.Regexp = regex - - return nil -} - -// MarshalText marshals regexp.Regexp as string -func (r *Regexp) MarshalText() ([]byte, error) { - if r.Regexp != nil { - return []byte(r.Regexp.String()), nil - } - - return nil, nil -} - -type Filter struct { - Name string `yaml:"name"` - Filters []FilterValue `yaml:"filters"` - Ignore []FilterValue `yaml:"ignore"` -} - -type FilterValue struct { - Value Regexp `yaml:"value"` - Score int `yaml:"score"` -} - -func GetGroupConfigsFromEmbed(lg *zap.SugaredLogger, dirName string, dir embed.FS) []Filter { - lg = lg.With("plugin", "slo", "phase", "init") - var res []Filter - fsys := fs.FS(dir) - yamlFs, err := fs.Sub(fsys, dirName) - if err != nil { - lg.Error(err) - lg.Debug(fmt.Sprintf("Debug error : yamlFs is %s", yamlFs)) - } - err = fs.WalkDir(yamlFs, ".", func(pathStr string, d fs.DirEntry, err error) error { - if !d.IsDir() { - fBytes, err := fs.ReadFile(dir, filepath.Join(dirName, d.Name())) - if err != nil { - lg.Error(err) - lg.Debug(fmt.Sprintf("Debug error : yamlFs is %s", yamlFs)) - panic(err) - } - var f Filter - err = yaml.Unmarshal(fBytes, &f) - if err != nil { - lg.Error(err) - lg.Debug(fmt.Sprintf("Debug error : yamlFs is %s", yamlFs)) - } - res = append(res, f) - } - return nil - }) - if err != nil { - lg.Error(err) - lg.Debug(fmt.Sprintf("Debug error : yamlFs is %s", yamlFs)) - } - return res -} - -func constructFilters(lg *zap.SugaredLogger) []Filter { - res := []Filter{} - for dirName, embedFs := range EnabledFilters { - filters := GetGroupConfigsFromEmbed(lg, dirName, embedFs) - res = append(res, filters...) - } - return res -} - -// Map metric -> group -> score -func scoredLabels(seriesInfo *cortexadmin.SeriesInfoList) *sloapi.MetricGroupList { - res := map[*cortexadmin.SeriesInfo]map[string]int{} - - for _, series := range seriesInfo.GetItems() { - series.GetSeriesName() - for _, groupname := range filters { - for _, matchFilter := range groupname.Filters { - if matchFilter.Value.MatchString(series.GetSeriesName()) { - if _, ok := res[series]; !ok { - res[series] = map[string]int{} - } - res[series][groupname.Name] += matchFilter.Score - } - } - - for _, ignoreFilter := range groupname.Ignore { - if ignoreFilter.Value.MatchString(series.GetSeriesName()) { - if _, ok := res[series]; !ok { - res[series] = map[string]int{} - } - res[series][groupname.Name] -= ignoreFilter.Score - } - } - } - } - groupedMetrics := sloapi.MetricGroupList{ - GroupNameToMetrics: map[string]*sloapi.MetricList{}, - } - for series, vals := range res { - groupName := "" - maxScore := 0 - for group, score := range vals { - if score > maxScore { - maxScore = score - groupName = group - } - } - if maxScore <= 0 { - groupName = "other metrics" - } - if _, ok := groupedMetrics.GroupNameToMetrics[groupName]; !ok { - groupedMetrics.GroupNameToMetrics[groupName] = &sloapi.MetricList{} - } - groupedMetrics.GroupNameToMetrics[groupName].Items = append(groupedMetrics.GroupNameToMetrics[groupName].Items, &sloapi.Metric{ - Id: series.GetSeriesName(), - Metadata: &sloapi.MetricMetadata{ - Description: series.Metadata.GetDescription(), - Unit: series.Metadata.GetUnit(), - Type: series.Metadata.GetType(), - }, - }) - } - return &groupedMetrics -} - -func ApplyFiltersToCortexEvents(seriesInfo *cortexadmin.SeriesInfoList) (*sloapi.MetricGroupList, error) { - return scoredLabels(seriesInfo), nil -} diff --git a/plugins/slo/pkg/slo/filters_test.go b/plugins/slo/pkg/slo/filters_test.go deleted file mode 100644 index 1b072abc3f..0000000000 --- a/plugins/slo/pkg/slo/filters_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package slo_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/rancher/opni/pkg/logger" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - "github.com/rancher/opni/plugins/slo/pkg/slo" -) - -var _ = Describe("SLO Filter tests", Ordered, Label("unit", "slow"), func() { - { - When("We use SLO filters", func() { - It("should get parse them from our embedded directory definitions", func() { - for dirName, embedFs := range slo.EnabledFilters { - filters := slo.GetGroupConfigsFromEmbed(logger.NewPluginLogger().Named("slo"), dirName, embedFs) - Expect(filters).NotTo(HaveLen(0)) - for _, filter := range filters { - Expect(filter.Name).NotTo(Equal("")) - Expect(filter.Filters).NotTo(BeEmpty()) - } - } - }) - - It("Should be able to score the events based on the filters", func() { - // array of prom metrics : label name -> label vals - series := &cortexadmin.SeriesInfoList{ - Items: []*cortexadmin.SeriesInfo{ - { - SeriesName: "go_gc_duration_seconds", - }, - { - SeriesName: "jvm_something", - }, - { - SeriesName: "jvm_something_else", - }, - { - SeriesName: "uptime_seconds", - }, - { - SeriesName: "kube-proxy", - }, - { - SeriesName: "apiserver_something", - }, - { - SeriesName: "request_duration_seconds", - }, - { - SeriesName: "cpu_usage_seconds_total", - }, - }, - } - - filteredGroups, err := slo.ApplyFiltersToCortexEvents(series) - Expect(err).To(Succeed()) - Expect(filteredGroups).NotTo(BeNil()) - Expect(filteredGroups.GroupNameToMetrics["golang metrics"].Items).To(HaveLen(1)) - Expect(filteredGroups.GroupNameToMetrics["jvm metrics"].Items).To(HaveLen(2)) - Expect(filteredGroups.GroupNameToMetrics["kubernetes metrics"].Items).To(HaveLen(1)) - Expect(filteredGroups.GroupNameToMetrics["network metrics"].Items).To(HaveLen(1)) - Expect(filteredGroups.GroupNameToMetrics["compute metrics"].Items).To(HaveLen(1)) - Expect(filteredGroups.GroupNameToMetrics["other metrics"].Items).To(HaveLen(2)) - }) - }) - } -}) diff --git a/plugins/slo/pkg/slo/impl.go b/plugins/slo/pkg/slo/impl.go deleted file mode 100644 index e21d132a73..0000000000 --- a/plugins/slo/pkg/slo/impl.go +++ /dev/null @@ -1,422 +0,0 @@ -package slo - -import ( - "context" - "fmt" - "strconv" - "sync" - "time" - - "slices" - - "emperror.dev/errors" - "github.com/google/uuid" - prommodel "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/rulefmt" - "github.com/prometheus/prometheus/promql/parser" - corev1 "github.com/rancher/opni/pkg/apis/core/v1" - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - "github.com/rancher/opni/pkg/util" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "github.com/samber/lo" - "github.com/tidwall/gjson" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/timestamppb" -) - -func (s *SLOMonitoring) WithCurrentRequest(ctx context.Context, req proto.Message) SLOStore { - s.req = req - s.ctx = ctx - return s -} - -func (s SLOMonitoring) Create() (*corev1.Reference, error) { - req := (s.req).(*sloapi.CreateSLORequest) - slo := CreateSLORequestToStruct(req) - rrecording, rmetadata, ralerting := slo.ConstructCortexRules(nil) - toApply := []rulefmt.RuleGroup{rrecording, rmetadata, ralerting} - ruleId := slo.GetId() - err := tryApplyThenDeleteCortexRules(s.ctx, s.p, s.p.logger, req.GetSlo().GetClusterId(), &ruleId, toApply) - if err != nil { - return nil, err - } - return &corev1.Reference{Id: slo.GetId()}, nil -} - -func (s SLOMonitoring) Update(existing *sloapi.SLOData) (*sloapi.SLOData, error) { - incomingSLO := (s.req).(*sloapi.SLOData) // Create is the same as Update if within the same cluster - newSlo := SLODataToStruct(incomingSLO) - rrecording, rmetadata, ralerting := newSlo.ConstructCortexRules(nil) - toApply := []rulefmt.RuleGroup{rrecording, rmetadata, ralerting} - err := tryApplyThenDeleteCortexRules(s.ctx, s.p, s.p.logger, incomingSLO.GetSLO().GetClusterId(), nil, toApply) - - // successfully applied rules to another cluster - if err == nil && existing.SLO.ClusterId != incomingSLO.SLO.ClusterId { - _, err := s.p.DeleteSLO(s.ctx, &corev1.Reference{Id: existing.Id}) - if err != nil { - s.lg.With("sloId", existing.Id).Error(fmt.Sprintf( - "Unable to delete SLO when updating between clusters : %v", - err)) - } - } - return incomingSLO, err -} - -func (s SLOMonitoring) Delete(existing *sloapi.SLOData) error { - id, clusterId := existing.Id, existing.SLO.ClusterId - //err := deleteCortexSLORules(s.p, id, clusterId, s.ctx, s.lg) - errArr := []error{} - slo := SLODataToStruct(existing) - rrecording, rmetadata, ralerting := slo.ConstructCortexRules(nil) - toApply := []rulefmt.RuleGroup{rrecording, rmetadata, ralerting} - for _, ruleName := range toApply { - for _, rule := range ruleName.Rules { - if rule.Alert.Value != "" { - err := deleteCortexSLORules( - s.ctx, - s.p, - s.p.logger, - clusterId, - rule.Alert.Value, - ) - if err != nil { - errArr = append(errArr, err) - } - } - if rule.Record.Value != "" { - err := deleteCortexSLORules( - s.ctx, - s.p, - s.p.logger, - clusterId, - rule.Record.Value, - ) - if err != nil { - errArr = append(errArr, err) - } - } - } - } - err := createGrafanaSLOMask(s.ctx, s.p, clusterId, id) - if err != nil { - s.p.logger.Errorf("creating grafana mask failed %s", err) - errArr = append(errArr, err) - } - return errors.Combine(errArr...) -} - -func (s SLOMonitoring) Clone(clone *sloapi.SLOData) (*corev1.Reference, *sloapi.SLOData, error) { - clonedData := util.ProtoClone(clone) - sloData := clonedData.GetSLO() - slo := SLODataToStruct(clonedData) - slo.SetId(uuid.New().String()) - slo.SetName(sloData.GetName() + "-clone") - rrecording, rmetadata, ralerting := slo.ConstructCortexRules(nil) - toApply := []rulefmt.RuleGroup{rrecording, rmetadata, ralerting} - ruleId := slo.GetId() - err := tryApplyThenDeleteCortexRules(s.ctx, s.p, s.p.logger, sloData.GetClusterId(), &ruleId, toApply) - clonedData.SLO.Name = sloData.Name + "-clone" - clonedData.Id = slo.GetId() - return &corev1.Reference{Id: slo.GetId()}, clonedData, err -} - -func (s SLOMonitoring) MultiClusterClone( - base *sloapi.SLOData, - inputClusters []*corev1.Reference, - svcBackend ServiceBackend, -) ([]*corev1.Reference, []*sloapi.SLOData, []error) { - clonedData := util.ProtoClone(base) - sloData := clonedData.GetSLO() - slo := SLODataToStruct(clonedData) - - clusters, err := s.p.mgmtClient.Get().ListClusters(s.ctx, &managementv1.ListClustersRequest{}) - if err != nil { - return nil, nil, []error{err} - } - var clusterIds []string - - for _, cluster := range clusters.Items { - clusterIds = append(clusterIds, cluster.Id) - } - clusterDefinitions := make([]*sloapi.SLOData, len(inputClusters)) - clusterIdsCreate := make([]*corev1.Reference, len(inputClusters)) - errArr := make([]error, len(inputClusters)) - var wg sync.WaitGroup - for idx, clusterId := range inputClusters { - wg.Add(1) - slo.SetId(uuid.New().String()) - slo.SetName(fmt.Sprintf("%s-clone-%d", sloData.GetName(), idx)) - rrecording, rmetadata, ralerting := slo.ConstructCortexRules(nil) - toApply := []rulefmt.RuleGroup{rrecording, rmetadata, ralerting} - // capture in closure - idx := idx - clusterId := clusterId - ruleId := slo.GetId() - if !slices.Contains(clusterIds, clusterId.Id) { - errArr[idx] = fmt.Errorf("cluster %s not found", clusterId.Id) - continue - } - svcBackend.WithCurrentRequest(s.ctx, &sloapi.ListServicesRequest{ - Datasource: "monitoring", - ClusterId: clusterId.Id, - }) - services, err := svcBackend.ListServices() - if err != nil { - errArr[idx] = err - continue - } - if services.ContainsId(sloData.GetServiceId()) { - errArr[idx] = fmt.Errorf("service %s not found on cluster %s", sloData.GetServiceId(), clusterId.Id) - continue - } - svcBackend.WithCurrentRequest(s.ctx, &sloapi.ListMetricsRequest{ - Datasource: "monitoring", - ClusterId: clusterId.Id, - ServiceId: sloData.GetServiceId(), - }) - metrics, err := svcBackend.ListMetrics() - if err != nil { - errArr[idx] = err - continue - } - if !metrics.ContainsId(sloData.GetGoodMetricName()) { - errArr[idx] = fmt.Errorf( - "good metric %s not found on cluster %s", - sloData.GetGoodMetricName(), - clusterId.Id, - ) - continue - } - if !metrics.ContainsId(sloData.GetTotalMetricName()) { - errArr[idx] = fmt.Errorf( - "total metric %s not found on cluster %s", - sloData.GetTotalMetricName(), - clusterId.Id, - ) - continue - } - errArr[idx] = tryApplyThenDeleteCortexRules(s.ctx, s.p, s.p.logger, clusterId.Id, &ruleId, toApply) - clonedData.SLO.Name = sloData.Name + "-clone-" + strconv.Itoa(idx) - clonedData.Id = slo.GetId() - clusterDefinitions[idx] = clonedData - clusterIdsCreate[idx] = &corev1.Reference{Id: slo.GetId()} - } - return clusterIdsCreate, clusterDefinitions, errArr -} - -// Status Only return errors here that should be considered severe InternalServerErrors -// - Check if enough time has passed to evaluate the rules -// - First Checks if it has NoData -// - If it has Data, check if it is within budget -// - If is within budget, check if any alerts are firing -func (s SLOMonitoring) Status(existing *sloapi.SLOData) (*sloapi.SLOStatus, error) { - now := time.Now() - - if now.Sub(existing.CreatedAt.AsTime()) <= sloapi.MinEvaluateInterval*2 { - s.lg.With("sloId", existing.Id).Debug("SLO status is not ready to be evaluated : ", - (&sloapi.SLOStatus{State: sloapi.SLOStatusState_Creating}).String()) - - return &sloapi.SLOStatus{State: sloapi.SLOStatusState_Creating}, nil - } - state := sloapi.SLOStatusState_Ok - slo := SLODataToStruct(existing) - // ======================= sli ======================= - sliErrorName := slo.ConstructRecordingRuleGroup(nil).Rules[0].Record - sliDataVector, err := QuerySLOComponentByRecordName( - s.ctx, - s.p.adminClient.Get(), - sliErrorName.Value, - existing.GetSLO().GetClusterId(), - ) - if err != nil { - return nil, err - } - if sliDataVector == nil || sliDataVector.Len() == 0 { - return &sloapi.SLOStatus{State: sloapi.SLOStatusState_NoData}, nil - } - s.lg.With("sloId", slo.GetId()).Debug("sli status response vector : ", sliDataVector.String()) - // ======================= error budget ======================= - // race condition can cause initial evaluation to fail with empty vector, resulting in no data state - // this is why we return creating state with two intervals - metadataBudgetRaw := slo.RawBudgetRemainingQuery() // this is not actually raw "raw", contains recording rule refs - metadataVector, err := QuerySLOComponentByRawQuery(s.ctx, s.p.adminClient.Get(), metadataBudgetRaw, existing.GetSLO().GetClusterId()) - if err != nil { - return nil, err - } - if metadataVector == nil || metadataVector.Len() == 0 { - return &sloapi.SLOStatus{State: sloapi.SLOStatusState_PartialDataOk}, nil - } - metadataBudget := (*metadataVector)[0].Value - if metadataBudget <= 0 { - return &sloapi.SLOStatus{State: sloapi.SLOStatusState_Breaching}, nil - } - s.lg.With("sloId", slo.GetId()).Debug("sli status ", metadataVector.String()) - // - //// ======================= alert ======================= - - alertBudgetRules := slo.ConstructAlertingRuleGroup(nil) - short, long := alertBudgetRules.Rules[0].Expr.Value, alertBudgetRules.Rules[1].Expr.Value - alertDataVector1, err := QuerySLOComponentByRawQuery(s.ctx, s.p.adminClient.Get(), short, existing.GetSLO().GetClusterId()) - if err != nil { - return nil, err - } - alertDataVector2, err := QuerySLOComponentByRawQuery(s.ctx, s.p.adminClient.Get(), long, existing.GetSLO().GetClusterId()) - if err != nil { - return nil, err - } - if alertDataVector1 == nil || alertDataVector1.Len() == 0 || alertDataVector2 == nil || alertDataVector2.Len() == 0 { - return &sloapi.SLOStatus{State: sloapi.SLOStatusState_PartialDataOk}, nil - } - if (*alertDataVector1)[len(*alertDataVector1)-1].Value > 0 || (*alertDataVector2)[len(*alertDataVector2)-1].Value > 0 { - return &sloapi.SLOStatus{State: sloapi.SLOStatusState_Warning}, nil - } - s.lg.With("sloId", slo.GetId()).Debug("alert status response vector ", alertDataVector1.String(), alertDataVector2.String()) - return &sloapi.SLOStatus{ - State: state, - }, nil -} - -func (s SLOMonitoring) Preview(slo *SLO) (*sloapi.SLOPreviewResponse, error) { - req := s.req.(*sloapi.CreateSLORequest) - preview := &sloapi.SLOPreviewResponse{ - PlotVector: &sloapi.PlotVector{ - Objective: normalizeObjective(req.GetSlo().GetTarget().GetValue()), - Items: []*sloapi.DataPoint{}, - Windows: []*sloapi.AlertFiringWindows{}, - }, - } - cur := time.Now() - dur, err := prommodel.ParseDuration(slo.sloPeriod) - if err != nil { - panic(err) - } - startTs, endTs := cur.Add(time.Duration(-dur)), cur - numSteps := 250 - step := time.Duration(endTs.Sub(startTs).Seconds()/float64(numSteps)) * time.Second - if step < time.Second { - step = time.Second - } - - ruleGroup := slo.ConstructRecordingRuleGroup(lo.ToPtr(sloapi.MinEvaluateInterval)) - sliPeriodErrorRate := ruleGroup.Rules[len(ruleGroup.Rules)-1].Expr.Value - sli := "1 - (max(" + sliPeriodErrorRate + ") OR on() vector(NaN))" // handles the empty case and still differentiates between 0 and empty - _, err = parser.ParseExpr(sli) - if err != nil { - panic(err) - } - sliDataMatrix, err := QuerySLOComponentByRawQueryRange(s.ctx, s.p.adminClient.Get(), - sli, req.GetSlo().GetClusterId(), - startTs, endTs, step, - ) - if err != nil { - return nil, err - } - for _, sample := range *sliDataMatrix { - for _, yieldedValue := range sample.Values { - ts := time.Unix(yieldedValue.Timestamp.Unix(), 0) - preview.PlotVector.Items = append(preview.PlotVector.Items, &sloapi.DataPoint{ - Timestamp: timestamppb.New(ts), - Sli: float64(yieldedValue.Value) * 100, - }) - } - } - - alertCriticalRawQuery, alertSevereRawQuery := slo.ConstructRawAlertQueries() - // ideally should be every 5 minutes for fine grained detail - // but for performance reasons, we will only query every 20 minutes - alertTimeStep := time.Minute * 20 - - alertWindowSevereMatrix, err := QuerySLOComponentByRawQueryRange(s.ctx, s.p.adminClient.Get(), - alertSevereRawQuery.Value, req.GetSlo().GetClusterId(), - startTs, endTs, alertTimeStep, - ) - if err != nil { - return nil, err - } - severeWindows, err := DetectActiveWindows("severe", alertWindowSevereMatrix) - if err != nil { - return nil, err - } - preview.PlotVector.Windows = append(preview.PlotVector.Windows, severeWindows...) - - alertWindowCriticalMatrix, err := QuerySLOComponentByRawQueryRange(s.ctx, s.p.adminClient.Get(), - alertCriticalRawQuery.Value, req.GetSlo().GetClusterId(), - startTs, endTs, step, - ) - if err != nil { - return nil, err - } - criticalWindows, err := DetectActiveWindows("critical", alertWindowCriticalMatrix) - if err != nil { - return nil, err - } - preview.PlotVector.Windows = append(preview.PlotVector.Windows, criticalWindows...) - return preview, nil -} - -func (m *MonitoringServiceBackend) WithCurrentRequest(ctx context.Context, req proto.Message) ServiceBackend { - m.req = req - m.ctx = ctx - return m -} - -func (m MonitoringServiceBackend) ListServices() (*sloapi.ServiceList, error) { - req := m.req.(*sloapi.ListServicesRequest) - res := &sloapi.ServiceList{} - discoveryQuery := `group by(job) ({__name__!=""})` - resp, err := m.p.adminClient.Get().Query( - m.ctx, - &cortexadmin.QueryRequest{ - Tenants: []string{req.GetClusterId()}, - Query: discoveryQuery, - }) - if err != nil { - return nil, err - } - result := gjson.Get(string(resp.Data), "data.result.#.metric.job") - if !result.Exists() { - return nil, fmt.Errorf("Could not convert prometheus service discovery to json ") - } - for _, v := range result.Array() { - res.Items = append(res.Items, &sloapi.Service{ - ClusterId: req.GetClusterId(), - ServiceId: v.String(), - }) - } - return res, nil -} - -func (m MonitoringServiceBackend) ListEvents() (*sloapi.EventList, error) { - res := &sloapi.EventList{ - Items: []*sloapi.Event{}, - } - req := (m.req).(*sloapi.ListEventsRequest) // Create is the same as Update if within the same cluster - resp, err := m.p.adminClient.Get().GetMetricLabelSets(m.ctx, &cortexadmin.LabelRequest{ - Tenant: req.GetClusterId(), - JobId: req.GetServiceId(), - MetricName: req.GetMetricId(), - }) - if err != nil { - return nil, err - } - for _, item := range resp.GetItems() { - res.Items = append(res.Items, &sloapi.Event{ - Key: item.GetName(), - Vals: item.GetItems(), - }) - } - return res, nil -} - -func (m MonitoringServiceBackend) ListMetrics() (*sloapi.MetricGroupList, error) { - req := (m.req).(*sloapi.ListMetricsRequest) // Create is the same as Update if within the same cluster - resp, err := m.p.adminClient.Get().GetSeriesMetrics(m.ctx, &cortexadmin.SeriesRequest{ - Tenant: req.GetClusterId(), - JobId: req.GetServiceId(), - }) - if err != nil { - return nil, err - } - return ApplyFiltersToCortexEvents(resp) -} diff --git a/plugins/slo/pkg/slo/interfaces.go b/plugins/slo/pkg/slo/interfaces.go deleted file mode 100644 index 7256108bc9..0000000000 --- a/plugins/slo/pkg/slo/interfaces.go +++ /dev/null @@ -1,92 +0,0 @@ -package slo - -import ( - "context" - "sync" - - corev1 "github.com/rancher/opni/pkg/apis/core/v1" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "go.uber.org/zap" - - "google.golang.org/protobuf/proto" -) - -var datasourceToSLO = make(map[string]SLOStore) -var datasourceToService = make(map[string]ServiceBackend) -var mu sync.Mutex - -func RegisterDatasource(datasource string, sloImpl SLOStore, serviceImpl ServiceBackend) { - defer mu.Unlock() - mu.Lock() - datasourceToSLO[datasource] = sloImpl - datasourceToService[datasource] = serviceImpl -} - -type SLOStore interface { - // This method has to handle storage of the SLO in the KVStore itself - // since there can be partial successes inside the method - Create() (*corev1.Reference, error) - Update(existing *sloapi.SLOData) (*sloapi.SLOData, error) - Delete(existing *sloapi.SLOData) error - Clone(clone *sloapi.SLOData) (*corev1.Reference, *sloapi.SLOData, error) - MultiClusterClone( - base *sloapi.SLOData, - clusters []*corev1.Reference, - svcBackend ServiceBackend, - ) ([]*corev1.Reference, []*sloapi.SLOData, []error) - Status(existing *sloapi.SLOData) (*sloapi.SLOStatus, error) - Preview(s *SLO) (*sloapi.SLOPreviewResponse, error) - WithCurrentRequest(ctx context.Context, req proto.Message) SLOStore -} -type ServiceBackend interface { - ListServices() (*sloapi.ServiceList, error) - ListMetrics() (*sloapi.MetricGroupList, error) - ListEvents() (*sloapi.EventList, error) - WithCurrentRequest(ctx context.Context, req proto.Message) ServiceBackend -} - -type MetricIds struct { - Good string - Total string -} - -type RequestBase struct { - req proto.Message - p *Plugin - ctx context.Context - lg *zap.SugaredLogger -} - -type SLOMonitoring struct { - RequestBase -} - -type SLOLogging struct { - RequestBase -} - -type MonitoringServiceBackend struct { - RequestBase -} - -func NewSLOMonitoringStore(p *Plugin, lg *zap.SugaredLogger) SLOStore { - return &SLOMonitoring{ - RequestBase{ - req: nil, - p: p, - ctx: context.Background(), - lg: lg, - }, - } -} - -func NewMonitoringServiceBackend(p *Plugin, lg *zap.SugaredLogger) ServiceBackend { - return &MonitoringServiceBackend{ - RequestBase{ - req: nil, - p: p, - ctx: context.TODO(), - lg: lg, - }, - } -} diff --git a/plugins/slo/pkg/slo/plugin.go b/plugins/slo/pkg/slo/plugin.go deleted file mode 100644 index 00bf7c3186..0000000000 --- a/plugins/slo/pkg/slo/plugin.go +++ /dev/null @@ -1,60 +0,0 @@ -package slo - -import ( - "context" - - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - "github.com/rancher/opni/plugins/slo/apis/slo" - "go.uber.org/zap" - - alertingv1 "github.com/rancher/opni/pkg/apis/alerting/v1" - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - "github.com/rancher/opni/pkg/logger" - managementext "github.com/rancher/opni/pkg/plugins/apis/apiextensions/management" - "github.com/rancher/opni/pkg/plugins/apis/system" - "github.com/rancher/opni/pkg/plugins/meta" - "github.com/rancher/opni/pkg/storage" - "github.com/rancher/opni/pkg/util" - "github.com/rancher/opni/pkg/util/future" -) - -type Plugin struct { - slo.UnsafeSLOServer - system.UnimplementedSystemPluginClient - - ctx context.Context - logger *zap.SugaredLogger - - storage future.Future[StorageAPIs] - mgmtClient future.Future[managementv1.ManagementClient] - adminClient future.Future[cortexadmin.CortexAdminClient] - alertEndpointClient future.Future[alertingv1.AlertEndpointsClient] -} - -type StorageAPIs struct { - SLOs storage.KeyValueStoreT[*slo.SLOData] - Services storage.KeyValueStoreT[*slo.Service] - Metrics storage.KeyValueStoreT[*slo.Metric] -} - -func NewPlugin(ctx context.Context) *Plugin { - return &Plugin{ - ctx: ctx, - logger: logger.NewPluginLogger().Named("slo"), - storage: future.New[StorageAPIs](), - mgmtClient: future.New[managementv1.ManagementClient](), - adminClient: future.New[cortexadmin.CortexAdminClient](), - alertEndpointClient: future.New[alertingv1.AlertEndpointsClient](), - } -} - -var _ slo.SLOServer = (*Plugin)(nil) - -func Scheme(ctx context.Context) meta.Scheme { - scheme := meta.NewScheme() - p := NewPlugin(ctx) - scheme.Add(system.SystemPluginID, system.NewPlugin(p)) - scheme.Add(managementext.ManagementAPIExtensionPluginID, - managementext.NewPlugin(util.PackService(&slo.SLO_ServiceDesc, p))) - return scheme -} diff --git a/plugins/slo/pkg/slo/simple.go b/plugins/slo/pkg/slo/simple.go deleted file mode 100644 index 7980035de0..0000000000 --- a/plugins/slo/pkg/slo/simple.go +++ /dev/null @@ -1,859 +0,0 @@ -package slo - -import ( - "bytes" - "fmt" - "os" - "strings" - "text/template" - "time" - - "github.com/google/uuid" - prommodel "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/rulefmt" - "github.com/prometheus/prometheus/promql/parser" - "github.com/rancher/opni/pkg/alerting/metrics" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "github.com/samber/lo" - "google.golang.org/protobuf/types/known/timestamppb" - "gopkg.in/yaml.v3" -) - -const ( - grafana_delete_mask = "slo_opni_delete" - // id labels - slo_uuid = "slo_opni_id" - slo_service = "slo_opni_service" - slo_name = "slo_opni_name" - slo_window = "slo_window" - - // recording rule names - slo_ratio_rate_query_name = "slo:sli_error:ratio_rate" - slo_alert_ticket_window = "slo:alert:ticket_window" - slo_alert_page_window = "slo:alert:page_window" - - // metadata rule names - slo_objective_ratio = "slo:objective:ratio" - slo_error_budget_ratio = "slo:error_budget:ratio" - slo_time_period_days = "slo:time_period:days" - slo_current_burn_rate_ratio = "slo:current_burn_rate:ratio" - slo_period_burn_rate_ratio = "slo:period_burn_rate:ratio" - slo_period_error_budget_remaining_ratio = "slo:period_error_budget_remaining:ratio" - slo_info = "opni_slo_info" - - // alert rule names - RecordingRuleSuffix = "-recording" - MetadataRuleSuffix = "-metadata" - AlertRuleSuffix = "-alerts" -) - -var ( - simpleQueryTpl = template.Must(template.New("query").Parse(`sum(rate({{.Metric}}{job="{{.JobId}}"{{.Labels}}}[{{.Window}}]))`)) - rawSliQueryTpl = template.Must(template.New("sliRawQuery").Parse(`1 - (({{.GoodQuery}})/({{.TotalQuery}}))`)) - sloFiltersTpl = template.Must(template.New("sloFilters").Parse(`{{.SloIdLabel}}="{{.SloId}}", {{.SloServiceLabel}}="{{.SloService}}", {{.SloNameLabel}}="{{.SloName}}"`)) -) - -// SliQueryInfo used for filling sli query templates -type SliQueryInfo struct { - GoodQuery string - TotalQuery string -} - -type SloFiltersInfo struct { - SloIdLabel string - SloServiceLabel string - SloNameLabel string - SloId string - SloService string - SloName string -} - -//type RuleGroupYAMLv2 struct { -// Name string `yaml:"name"` -// Interval prommodel.Duration `yaml:"interval,omitempty"` -// Rules []rulefmt.Rule `yaml:"rules"` -//} - -type LabelPair struct { - Key string - Vals []string -} - -type LabelPairs []LabelPair - -func (l LabelPairs) Construct() string { // kinda hacky & technically unoptimized but works - if len(l) == 0 { - return "" - } - s := "" - for _, labelPair := range l { - if labelPair.Key == "" { - continue - } - orCompositionVal := "" - if len(labelPair.Vals) == 1 { - orCompositionVal += labelPair.Vals[0] - } else { - for idx, val := range labelPair.Vals { - orCompositionVal += val - if idx != len(labelPair.Vals)-1 { - orCompositionVal += "|" - } - } - } - s += fmt.Sprintf(",%s=~\"%s\"", labelPair.Key, orCompositionVal) - } - return s -} - -type IdentificationLabels map[string]string -type UserLabels []string -type Service string -type Metric string - -type SLO struct { - sloPeriod string - objective float64 // 0 < x < 100 - svc Service - goodMetric Metric - totalMetric Metric - idLabels IdentificationLabels - userLabels map[string]string - goodEvents LabelPairs - totalEvents LabelPairs -} - -func normalizeObjective(objective float64) float64 { - return objective / 100 -} - -func NewSLO( - sloName string, - sloPeriod string, - objective float64, - svc Service, - goodMetric Metric, - totalMetric Metric, - userLabels map[string]string, - goodEvents []LabelPair, - totalEvents []LabelPair, -) *SLO { - newId := uuid.New().String() - ilabels := IdentificationLabels{slo_uuid: newId, slo_name: sloName, slo_service: string(svc)} - - return &SLO{ - svc: svc, - sloPeriod: sloPeriod, - goodMetric: goodMetric, - totalMetric: totalMetric, - userLabels: userLabels, - goodEvents: goodEvents, - totalEvents: totalEvents, - idLabels: ilabels, - objective: objective, - } -} - -func SLOFromId( - sloName string, - sloPeriod string, - objective float64, - svc Service, - goodMetric Metric, - totalMetric Metric, - userLabels map[string]string, - goodEvents []LabelPair, - totalEvents []LabelPair, - id string, -) *SLO { - ilabels := IdentificationLabels{slo_uuid: id, slo_name: sloName, slo_service: string(svc)} - - return &SLO{ - svc: svc, - goodMetric: goodMetric, - totalMetric: totalMetric, - sloPeriod: sloPeriod, - userLabels: userLabels, - goodEvents: goodEvents, - totalEvents: totalEvents, - idLabels: ilabels, - objective: objective, - } -} - -func CreateSLORequestToStruct(c *sloapi.CreateSLORequest) *SLO { - if c.Slo.GetGoodMetricName() == c.Slo.GetTotalMetricName() { - c.Slo.GoodEvents, c.Slo.TotalEvents = ToMatchingSubsetIdenticalMetric(c.Slo.GoodEvents, c.Slo.TotalEvents) - } - reqSLO := c.Slo - userLabels := reqSLO.GetLabels() - sloLabels := map[string]string{} - for _, label := range userLabels { - sloLabels[label.GetName()] = "true" - } - goodEvents := []LabelPair{} - for _, goodEvent := range reqSLO.GetGoodEvents() { - goodEvents = append(goodEvents, LabelPair{ - Key: goodEvent.GetKey(), - Vals: goodEvent.GetVals(), - }) - } - totalEvents := []LabelPair{} - for _, totalEvent := range reqSLO.GetTotalEvents() { - totalEvents = append(totalEvents, LabelPair{ - Key: totalEvent.GetKey(), - Vals: totalEvent.GetVals(), - }) - } - return NewSLO( - reqSLO.GetName(), - reqSLO.GetSloPeriod(), - reqSLO.GetTarget().GetValue(), - Service(reqSLO.GetServiceId()), - Metric(reqSLO.GetGoodMetricName()), - Metric(reqSLO.GetTotalMetricName()), - sloLabels, - goodEvents, - totalEvents, - ) -} - -func SLODataToStruct(s *sloapi.SLOData) *SLO { - reqSLO := s.SLO - if reqSLO.GetGoodMetricName() == reqSLO.GetTotalMetricName() { - reqSLO.GoodEvents, reqSLO.TotalEvents = ToMatchingSubsetIdenticalMetric(reqSLO.GoodEvents, reqSLO.TotalEvents) - } - userLabels := reqSLO.GetLabels() - sloLabels := map[string]string{} - for _, label := range userLabels { - sloLabels[label.GetName()] = "true" - } - goodEvents := []LabelPair{} - for _, goodEvent := range reqSLO.GetGoodEvents() { - goodEvents = append(goodEvents, LabelPair{ - Key: goodEvent.GetKey(), - Vals: goodEvent.GetVals(), - }) - } - totalEvents := []LabelPair{} - for _, totalEvent := range reqSLO.GetTotalEvents() { - totalEvents = append(totalEvents, LabelPair{ - Key: totalEvent.GetKey(), - Vals: totalEvent.GetVals(), - }) - } - if s.Id == "" { - return NewSLO( - reqSLO.GetName(), - reqSLO.GetSloPeriod(), - reqSLO.GetTarget().GetValue(), - Service(reqSLO.GetServiceId()), - Metric(reqSLO.GetGoodMetricName()), - Metric(reqSLO.GetTotalMetricName()), - sloLabels, - goodEvents, - totalEvents, - ) - } - return SLOFromId( - reqSLO.GetName(), - reqSLO.GetSloPeriod(), - reqSLO.GetTarget().GetValue(), - Service(reqSLO.GetServiceId()), - Metric(reqSLO.GetGoodMetricName()), - Metric(reqSLO.GetTotalMetricName()), - sloLabels, - goodEvents, - totalEvents, - s.Id, - ) -} - -func (s *SLO) GetId() string { - return s.idLabels[slo_uuid] // let it panic if not found -} - -func (s *SLO) SetId(id string) { - s.idLabels[slo_uuid] = id -} - -func (s *SLO) GetName() string { - return s.idLabels[slo_name] // let it panic if not found -} - -func (s *SLO) SetName(input string) { - s.idLabels[slo_name] = input -} - -func (s *SLO) GetPeriod() string { - return s.sloPeriod -} - -func (s *SLO) GetObjective() float64 { - return s.objective -} - -func (s *SLO) GetPrometheusRuleFilterByIdLabels() (string, error) { - var b bytes.Buffer - err := sloFiltersTpl.Execute(&b, SloFiltersInfo{ - SloIdLabel: slo_uuid, - SloServiceLabel: slo_service, - SloNameLabel: slo_name, - SloId: s.GetId(), - SloService: string(s.svc), - SloName: s.GetName(), - }) - return b.String(), err -} - -func (s *SLO) RawObjectiveQuery() string { - return fmt.Sprintf("vector(%.9f)", normalizeObjective(s.objective)) -} - -func (s *SLO) RawErrorBudgetQuery() string { - return fmt.Sprintf("vector(1-%.9f)", normalizeObjective(s.objective)) -} - -// RawCurrentBurnRateQuery -// ratioRate : slo:sli_error:ratio_rate -func (s *SLO) RawCurrentBurnRateQuery() string { - ruleFilters, err := s.GetPrometheusRuleFilterByIdLabels() - if err != nil { - panic(err) - } - fastWindow := NewWindowRange(s.sloPeriod)[0] - strRes := (slo_ratio_rate_query_name + fastWindow) + "{" + ruleFilters + "}" + " / " + - fmt.Sprintf("on(%s, %s, %s)", slo_uuid, slo_service, slo_name) + " group_left\n" + - slo_error_budget_ratio + "{" + ruleFilters + "}" - - return strRes -} - -func (s *SLO) RawPeriodBurnRateQuery() string { - ruleFilters, err := s.GetPrometheusRuleFilterByIdLabels() - if err != nil { - panic(err) - } - slowestWindow := NewWindowRange(s.sloPeriod)[len(NewWindowRange(s.sloPeriod))-1] - strRes := (slo_ratio_rate_query_name + slowestWindow) + "{" + ruleFilters + "}" + " / " + - fmt.Sprintf("on(%s, %s, %s)", slo_uuid, slo_service, slo_name) + " group_left\n" + - slo_error_budget_ratio + "{" + ruleFilters + "}" - return strRes -} - -func (s *SLO) RawPeriodDurationQuery() string { - dur, err := prommodel.ParseDuration(s.sloPeriod) - if err != nil { - panic(err) - } - timeDur := time.Duration(dur) - days := int(timeDur.Hours() / 24) - return fmt.Sprintf("vector(%d)", days) -} - -func (s *SLO) RawBudgetRemainingQuery() string { - ruleFilters, err := s.GetPrometheusRuleFilterByIdLabels() - if err != nil { - panic(err) - } - return "1 - " + slo_period_burn_rate_ratio + "{" + ruleFilters + "}" -} - -func (s *SLO) RawDashboardInfoQuery() string { - return "vector(1)" -} - -func (s *SLO) RawGoodEventsQuery(w string) (string, error) { - goodConstructedEvents := s.goodEvents.Construct() - var bGood bytes.Buffer - err := simpleQueryTpl.Execute(&bGood, map[string]string{ - "Metric": string(s.goodMetric), - "JobId": string(s.svc), - "Labels": goodConstructedEvents, - "Window": w, - }) - - return bGood.String(), err -} - -func (s *SLO) RawTotalEventsQuery(w string) (string, error) { - totalConstructedEvents := s.totalEvents.Construct() - var bTotal bytes.Buffer - err := simpleQueryTpl.Execute(&bTotal, map[string]string{ - "Metric": string(s.totalMetric), - "JobId": string(s.svc), - "Labels": totalConstructedEvents, - "Window": w, - }) - return bTotal.String(), err -} - -func (s *SLO) RawSLIQuery(w string) (string, error) { - good, err := s.RawGoodEventsQuery(w) - if err != nil { - return "", err - } - total, err := s.RawTotalEventsQuery(w) - if err != nil { - return "", err - } - var bQuery bytes.Buffer - err = rawSliQueryTpl.Execute(&bQuery, SliQueryInfo{ - GoodQuery: good, - TotalQuery: total, - }) - return bQuery.String(), err -} - -func (s *SLO) ConstructRecordingRuleGroup(interval *time.Duration) rulefmt.RuleGroup { - var promInterval prommodel.Duration - var err error - if interval == nil { //sensible default is 1m - promInterval, err = prommodel.ParseDuration(TimeDurationToPromStr(time.Minute)) - - } else { - promInterval, err = prommodel.ParseDuration(TimeDurationToPromStr(*interval)) - if err != nil { - panic(err) - } - } - if err != nil { - panic(err) - } - rrecording := rulefmt.RuleGroup{ - Name: s.GetId() + RecordingRuleSuffix, - Interval: promInterval, - } - for _, w := range NewWindowRange(s.sloPeriod) { - rawSli, err := s.RawSLIQuery(w) - if err != nil { - panic(err) - } - rrecording.Rules = append(rrecording.Rules, rulefmt.RuleNode{ - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_ratio_rate_query_name + w, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: rawSli, - }, - Labels: MergeLabels(s.idLabels, map[string]string{ - slo_window: w, - }, - s.userLabels, - ), - }) - } - return rrecording -} - -func (s *SLO) ConstructMetadataRules(interval *time.Duration) rulefmt.RuleGroup { - var promInterval prommodel.Duration - var err error - if interval == nil { //sensible default is 1m - promInterval, err = prommodel.ParseDuration(TimeDurationToPromStr(time.Minute)) - - } else { - promInterval, err = prommodel.ParseDuration(TimeDurationToPromStr(*interval)) - } - if err != nil { - panic(err) - } - rmetadata := rulefmt.RuleGroup{ - Name: s.GetId() + MetadataRuleSuffix, - Interval: promInterval, - } - rmetadata.Rules = []rulefmt.RuleNode{ - { - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_objective_ratio, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: s.RawObjectiveQuery(), - }, - Labels: MergeLabels( - s.idLabels, - s.userLabels, - ), - }, - { - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_error_budget_ratio, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: s.RawErrorBudgetQuery(), - }, - Labels: MergeLabels( - s.idLabels, - s.userLabels, - ), - }, - { - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_time_period_days, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: s.RawPeriodDurationQuery(), - }, - Labels: MergeLabels( - s.idLabels, - s.userLabels, - ), - }, - { - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_current_burn_rate_ratio, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: s.RawCurrentBurnRateQuery(), - }, - Labels: MergeLabels( - s.idLabels, - s.userLabels, - ), - }, - { - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_period_burn_rate_ratio, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: s.RawPeriodBurnRateQuery(), - }, - Labels: MergeLabels( - s.idLabels, - s.userLabels, - ), - }, - { - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_period_error_budget_remaining_ratio, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: s.RawBudgetRemainingQuery(), - }, - Labels: MergeLabels( - s.idLabels, - s.userLabels, - ), - }, - { - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_info, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: s.RawDashboardInfoQuery(), - }, - Labels: MergeLabels( - s.idLabels, - s.userLabels, - ), - }, - } - return rmetadata -} - -func (s *SLO) AlertPageThreshold() float64 { - return 0.5 -} - -// ConstructAlertingRuleGroup -// -// Note: first two are expected to be the recording rules -// Note: second two are expected to be the alerting rules -func (s *SLO) ConstructAlertingRuleGroup(interval *time.Duration) rulefmt.RuleGroup { - var promInterval prommodel.Duration - var err error - if interval == nil { //sensible default is 1m - promInterval, err = prommodel.ParseDuration(TimeDurationToPromStr(time.Minute)) - - } else { - promInterval, err = prommodel.ParseDuration(TimeDurationToPromStr(*interval)) - } - if err != nil { - panic(err) - } - ralerting := rulefmt.RuleGroup{ - Name: s.GetId() + AlertRuleSuffix, - Interval: promInterval, - } - sloFilters, err := s.GetPrometheusRuleFilterByIdLabels() - if err != nil { - panic(err) - } - sloFilters = "{" + sloFilters + "}" - dur, err := prommodel.ParseDuration(s.sloPeriod) - if err != nil { - panic(err) - } - mwmbWindow := GenerateGoogleWindows(time.Duration(dur)) - var exprTicket bytes.Buffer - errorBudgetRatio := 100 - s.objective - // if errorBudgetRatio == 0 { - //panic(fmt.Sprintf("error budget ratio cannot be treated as 0, from objective : %.9f", s.objective)) - // } - err = mwmbAlertTplBool.Execute(&exprTicket, map[string]string{ - "WindowLabel": slo_window, - "QuickShortMetric": slo_ratio_rate_query_name + "5m", - "QuickShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketQuick()), - "QuickLongMetric": slo_ratio_rate_query_name + "30m", - "QuickLongBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketSlow()), - "SlowShortMetric": slo_ratio_rate_query_name + "2h", - "SlowShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketQuick()), - "SlowQuickMetric": slo_ratio_rate_query_name + "6h", - "SlowQuickBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketSlow()), - "ErrorBudgetRatio": fmt.Sprintf("%.9f", errorBudgetRatio), - "MetricFilter": sloFilters, - }) - if err != nil { - panic(err) - } - var exprPage bytes.Buffer - err = mwmbAlertTplBool.Execute(&exprPage, map[string]string{ - "WindowLabel": slo_window, - "QuickShortMetric": slo_ratio_rate_query_name + "5m", - "QuickShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageQuick()), - "QuickLongMetric": slo_ratio_rate_query_name + "30m", - "QuickLongBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageSlow()), - "SlowShortMetric": slo_ratio_rate_query_name + "2h", - "SlowShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageQuick()), - "SlowQuickMetric": slo_ratio_rate_query_name + "6h", - "SlowQuickBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageSlow()), - "ErrorBudgetRatio": fmt.Sprintf("%.9f", errorBudgetRatio), - "MetricFilter": sloFilters, - }) - if err != nil { - panic(err) - } - var recordTicket bytes.Buffer - err = mwmbAlertTplBool.Execute(&recordTicket, map[string]string{ - "WindowLabel": slo_window, - "QuickShortMetric": slo_ratio_rate_query_name + "5m", - "QuickShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketQuick()), - "QuickLongMetric": slo_ratio_rate_query_name + "30m", - "QuickLongBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketSlow()), - "SlowShortMetric": slo_ratio_rate_query_name + "2h", - "SlowShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketQuick()), - "SlowQuickMetric": slo_ratio_rate_query_name + "6h", - "SlowQuickBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedTicketSlow()), - "ErrorBudgetRatio": fmt.Sprintf("%.9f", errorBudgetRatio), - "MetricFilter": sloFilters, - }) - if err != nil { - panic(err) - } - var recordPage bytes.Buffer - err = mwmbAlertTplBool.Execute(&recordPage, map[string]string{ - "WindowLabel": slo_window, - "QuickShortMetric": slo_ratio_rate_query_name + "5m", - "QuickShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageQuick()), - "QuickLongMetric": slo_ratio_rate_query_name + "30m", - "QuickLongBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageSlow()), - "SlowShortMetric": slo_ratio_rate_query_name + "2h", - "SlowShortBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageQuick()), - "SlowQuickMetric": slo_ratio_rate_query_name + "6h", - "SlowQuickBurnFactor": fmt.Sprintf("%.9f", mwmbWindow.GetSpeedPageSlow()), - "ErrorBudgetRatio": fmt.Sprintf("%.9f", errorBudgetRatio), - "MetricFilter": sloFilters, - }) - if err != nil { - panic(err) - } - - // Note: first two are expected to be the recording rules - ralerting.Rules = append(ralerting.Rules, rulefmt.RuleNode{ - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_alert_ticket_window, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: recordTicket.String(), - }, - Labels: MergeLabels(s.idLabels, map[string]string{"slo_severity": "ticket"}, s.userLabels), - }) - ralerting.Rules = append(ralerting.Rules, rulefmt.RuleNode{ - Record: yaml.Node{ - Kind: yaml.ScalarNode, - Value: slo_alert_page_window, - }, - Expr: yaml.Node{ - Kind: yaml.ScalarNode, - Value: recordPage.String(), - }, - Labels: MergeLabels(s.idLabels, map[string]string{"slo_severity": "page"}, s.userLabels), - }) - - // note: second two are expected to be the alerting rules - arPage := metrics.AlertingRule{ - Expr: exprPage.String(), - Labels: MergeLabels(s.idLabels, map[string]string{"slo_severity": "page"}, s.userLabels), - } - arTicket := metrics.AlertingRule{ - Expr: exprTicket.String(), - Labels: MergeLabels(s.idLabels, map[string]string{"slo_severity": "ticket"}, s.userLabels), - } - arPageRule, err := arPage.Build(metrics.WithSloId(s.GetId(), "page", AlertRuleSuffix)) - if err != nil { - panic(err) - } - - arTicketRule, err := arTicket.Build(metrics.WithSloId(s.GetId(), "ticket", AlertRuleSuffix)) - if err != nil { - panic(err) - } - ralerting.Rules = append(ralerting.Rules, *arPageRule) - ralerting.Rules = append(ralerting.Rules, *arTicketRule) - return ralerting -} - -func (s *SLO) ConstructCortexRules(interval *time.Duration) (sli, metadata, alerts rulefmt.RuleGroup) { - rrecording := s.ConstructRecordingRuleGroup(interval) - rmetadata := s.ConstructMetadataRules(interval) - ralerts := s.ConstructAlertingRuleGroup(interval) - - _, debug := os.LookupEnv("DEBUG_RULES") - if debug { - srecording, err := yaml.Marshal(rrecording) - if err != nil { - panic(err) - } - smetadata, err := yaml.Marshal(rmetadata) - if err != nil { - panic(err) - } - salerts, err := yaml.Marshal(ralerts) - if err != nil { - panic(err) - } - os.WriteFile(s.GetId()+"-recording.yaml", srecording, 0644) - os.WriteFile(s.GetId()+"-metadata.yaml", smetadata, 0644) - os.WriteFile(s.GetId()+"-alerts.yaml", salerts, 0644) - } - return rrecording, rmetadata, ralerts -} - -func (s *SLO) ConstructRawAlertQueries() (yaml.Node, yaml.Node) { - filters, err := s.GetPrometheusRuleFilterByIdLabels() - if err != nil { - panic(err) - } - filters = "{" + filters + "}" - alertCriticalRawQuery := s.ConstructAlertingRuleGroup(lo.ToPtr(sloapi.MinEvaluateInterval)).Rules[0].Expr - alertCriticalRawQuery.Value = strings.Replace(alertCriticalRawQuery.Value, filters, "", -1) - alertCriticalRawQuery.Value = strings.Replace(alertCriticalRawQuery.Value, fmt.Sprintf("without (%s)", slo_window), "", -1) - alertSevereRawQuery := s.ConstructAlertingRuleGroup(lo.ToPtr(sloapi.MinEvaluateInterval)).Rules[1].Expr - alertSevereRawQuery.Value = strings.Replace(alertSevereRawQuery.Value, filters, "", -1) - alertSevereRawQuery.Value = strings.Replace(alertSevereRawQuery.Value, fmt.Sprintf("without (%s)", slo_window), "", -1) - - for _, rule := range s.ConstructRecordingRuleGroup(nil).Rules { - alertCriticalRawQuery.Value = strings.Replace(alertCriticalRawQuery.Value, rule.Record.Value, rule.Expr.Value, -1) - alertSevereRawQuery.Value = strings.Replace(alertSevereRawQuery.Value, rule.Record.Value, rule.Expr.Value, -1) - } - - _, err = parser.ParseExpr(alertCriticalRawQuery.Value) - if err != nil { - panic(err) - } - _, err = parser.ParseExpr(alertSevereRawQuery.Value) - if err != nil { - panic(err) - } - return alertCriticalRawQuery, alertSevereRawQuery -} - -func NewWindowRange(sloPeriod string) []string { - return []string{"5m", "30m", "1h", "2h", "6h", "1d", sloPeriod} -} - -// DetectActiveWindows -// -// @warning Expectation is that the timestamps are ordered when traversing -// matrix --> sample streams --> [] values -// but this may not always be the case -func DetectActiveWindows(severity string, matrix *prommodel.Matrix) ([]*sloapi.AlertFiringWindows, error) { - returnWindows := []*sloapi.AlertFiringWindows{} - if matrix == nil { - return nil, fmt.Errorf("Got empty alerting window %s matrix from cortex", severity) - } - for _, row := range *matrix { - for _, rowValue := range row.Values { - ts := time.Unix(rowValue.Timestamp.Unix(), 0) - // rising edge - if rowValue.Value == 1 { - if len(returnWindows) == 0 { - returnWindows = append(returnWindows, &sloapi.AlertFiringWindows{ - Start: timestamppb.New(ts), - End: nil, - Severity: severity, - }) - } else { - if returnWindows[len(returnWindows)-1].End != nil { - returnWindows = append(returnWindows, &sloapi.AlertFiringWindows{ - Start: timestamppb.New(ts), - End: nil, - Severity: severity, - }) - } - } - } else { //falling edge - if len(returnWindows) > 0 && returnWindows[len(returnWindows)-1].End == nil { - returnWindows[len(returnWindows)-1].End = timestamppb.New(ts) - } - } - } - } - if len(returnWindows) > 0 { - if returnWindows[len(returnWindows)-1].End == nil { - returnWindows[len(returnWindows)-1].End = timestamppb.New(time.Now()) - } - } - return returnWindows, nil -} - -// ToMatchingSubsetIdenticalMetric only applies when the good metric & total metric id is the same -func ToMatchingSubsetIdenticalMetric(goodEvents, totalEvents []*sloapi.Event) (good, total []*sloapi.Event) { - indexGood := map[string][]string{} - for _, g := range goodEvents { - indexGood[g.Key] = g.Vals - } - for _, t := range totalEvents { - if t.Key == "" || t.Vals == nil { - continue - } - if _, ok := indexGood[t.Key]; ok { // event type defined on good and total so reconcile - t.Vals = LeftJoinSlice(indexGood[t.Key], t.Vals) - } else { - // event type not defined on good, so coerce good to have it, otherwise - goodEvents = append(goodEvents, t) - } - } - return goodEvents, totalEvents -} - -func MergeRuleGroups(left rulefmt.RuleGroup, right *rulefmt.RuleGroup) *rulefmt.RuleGroup { - newRules := LeftJoinSliceAbstract[rulefmt.RuleNode, string]( - left.Rules, - right.Rules, - func(r rulefmt.RuleNode) string { return r.Record.Value }, - ) - return &rulefmt.RuleGroup{ - Name: left.Name, - Interval: left.Interval, - Rules: newRules, - } -} diff --git a/plugins/slo/pkg/slo/slo_suite_test.go b/plugins/slo/pkg/slo/slo_suite_test.go deleted file mode 100644 index 282a64f7e5..0000000000 --- a/plugins/slo/pkg/slo/slo_suite_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package slo_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - _ "github.com/rancher/opni/pkg/test/setup" - _ "github.com/rancher/opni/plugins/alerting/test" - _ "github.com/rancher/opni/plugins/metrics/test" -) - -func TestSlo(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Slo Suite") -} diff --git a/plugins/slo/pkg/slo/slo_test.go b/plugins/slo/pkg/slo/slo_test.go deleted file mode 100644 index d7bbbbec8e..0000000000 --- a/plugins/slo/pkg/slo/slo_test.go +++ /dev/null @@ -1,826 +0,0 @@ -package slo_test - -import ( - "context" - "fmt" - "math/rand" - "strings" - "time" - - "github.com/goombaio/namegenerator" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - prommodel "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/rulefmt" - "github.com/prometheus/prometheus/promql/parser" - capabilityv1 "github.com/rancher/opni/pkg/apis/capability/v1" - corev1 "github.com/rancher/opni/pkg/apis/core/v1" - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - storagev1 "github.com/rancher/opni/pkg/apis/storage/v1" - - "github.com/rancher/opni/pkg/capabilities/wellknown" - "github.com/rancher/opni/pkg/metrics/compat" - "github.com/rancher/opni/pkg/test" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - "github.com/rancher/opni/plugins/metrics/apis/cortexops" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "github.com/rancher/opni/plugins/slo/pkg/slo" - "google.golang.org/protobuf/types/known/durationpb" - "google.golang.org/protobuf/types/known/emptypb" - "gopkg.in/yaml.v3" -) - -// need to check always that good <= total -func expectValidEventSubsets(good []*sloapi.Event, total []*sloapi.Event) { - cacheTotal := make(map[string]bool) - listVals := make(map[string][]string) - for i := 0; i < len(good); i++ { - listVals[good[i].Key] = good[i].Vals - } - for i := 0; i < len(total); i++ { - cacheTotal[total[i].Key] = false - Expect(len(total[i].Vals)).To(BeNumerically(">=", len(listVals[total[i].Key]))) - Expect(total[i].Vals).To(ContainElements(listVals[total[i].Key])) // total events must be at least a super set of good events - } - for i := 0; i < len(good); i++ { - if _, ok := cacheTotal[good[i].Key]; ok { - cacheTotal[good[i].Key] = true - } - } - - // everything in total should be defined in good - for _, v := range cacheTotal { - Expect(v).To(BeTrue()) - } - -} - -var _ = Describe("Converting SLO information to Cortex rules", Ordered, Label("integration", "slow"), func() { - sloObj := slo.NewSLO( - "slo-name", - "30d", - 99.99, - "prometheus", - "prometheus_http_requests_total", - "prometheus_http_requests_total", - map[string]string{ - "important": "true", - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200"}, - }, - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200", "500", "503"}, - }, - }, - ) - // test environment references - var env *test.Environment - var pPort int - var adminClient cortexadmin.CortexAdminClient - - BeforeAll(func() { - env = &test.Environment{} - Expect(env.Start()).To(Succeed()) - DeferCleanup(env.Stop) - - client := env.NewManagementClient() - token, err := client.CreateBootstrapToken(env.Context(), &managementv1.CreateBootstrapTokenRequest{ - Ttl: durationpb.New(time.Hour), - }) - Expect(err).NotTo(HaveOccurred()) - info, err := client.CertsInfo(env.Context(), &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - opsClient := cortexops.NewCortexOpsClient(env.ManagementClientConn()) - _, err = opsClient.ConfigureCluster(context.Background(), &cortexops.ClusterConfiguration{ - Mode: cortexops.DeploymentMode_AllInOne, - Storage: &storagev1.StorageSpec{ - Backend: storagev1.Filesystem, - }, - }) - Expect(err).NotTo(HaveOccurred()) - - _, errC := env.StartAgent("agent", token, []string{info.Chain[len(info.Chain)-1].Fingerprint}, test.WithContext(env.Context())) - Eventually(errC).Should(Receive(BeNil())) - _, errC = env.StartAgent("agent2", token, []string{info.Chain[len(info.Chain)-1].Fingerprint}, test.WithContext(env.Context())) - Eventually(errC).Should(Receive(BeNil())) - - _, err = client.InstallCapability(env.Context(), &managementv1.CapabilityInstallRequest{ - Name: wellknown.CapabilityMetrics, - Target: &capabilityv1.InstallRequest{ - Cluster: &corev1.Reference{Id: "agent"}, - }, - }) - - Expect(err).NotTo(HaveOccurred()) - _, err = client.InstallCapability(env.Context(), &managementv1.CapabilityInstallRequest{ - Name: wellknown.CapabilityMetrics, - Target: &capabilityv1.InstallRequest{ - Cluster: &corev1.Reference{Id: "agent2"}, - }, - }) - Expect(err).NotTo(HaveOccurred()) - - adminClient = cortexadmin.NewCortexAdminClient(env.ManagementClientConn()) - Eventually(func() error { - stats, err := adminClient.AllUserStats(context.Background(), &emptypb.Empty{}) - if err != nil { - return err - } - for _, item := range stats.Items { - if item.UserID == "agent" { - if item.NumSeries > 0 { - return nil - } - } - } - return fmt.Errorf("waiting for metric data to be stored in cortex") - }, 30*time.Second, 1*time.Second).Should(Succeed()) - Eventually(func() error { - stats, err := adminClient.AllUserStats(context.Background(), &emptypb.Empty{}) - if err != nil { - return err - } - for _, item := range stats.Items { - if item.UserID == "agent2" { - if item.NumSeries > 0 { - return nil - } - } - } - return fmt.Errorf("waiting for metric data to be stored in cortex") - }, 30*time.Second, 1*time.Second).Should(Succeed()) - }) - - When("We receive alert matrix data from cortex", func() { - It("should be able to convert the alert matrix data to active windows", func() { - _, err := slo.DetectActiveWindows("severe", nil) - Expect(err).To(HaveOccurred()) - - start := time.Now().Add(-(time.Hour * 24)) - startAlertWindow1 := time.Now().Add(-time.Hour * 5) - endAlertWindow1 := time.Now().Add(-time.Hour * 4) - startAlertWindow2 := time.Now().Add(-time.Hour * 3) - endAlertWindow2 := time.Now().Add(-time.Hour * 2) - - matrix := &prommodel.Matrix{ - // sample stream 1 - { - Values: []prommodel.SamplePair{ - { - Timestamp: prommodel.TimeFromUnix(start.Unix()), - Value: 0, - }, - { - Timestamp: prommodel.TimeFromUnix(startAlertWindow1.Unix()), - Value: 1, - }, - }, - }, - // sample stream 2 - { - Values: []prommodel.SamplePair{ - { - Timestamp: prommodel.TimeFromUnix(endAlertWindow1.Unix()), - Value: 0, - }, - { - Timestamp: prommodel.TimeFromUnix(endAlertWindow1.Add(time.Minute).Unix()), - Value: 0, - }, - { - Timestamp: prommodel.TimeFromUnix(startAlertWindow2.Unix()), - Value: 1, - }, - { - Timestamp: prommodel.TimeFromUnix(startAlertWindow2.Add(time.Minute).Unix()), - Value: 1, - }, - { - Timestamp: prommodel.TimeFromUnix(endAlertWindow2.Unix()), - Value: 0, - }, - }, - }, - } - windows, err := slo.DetectActiveWindows("severe", matrix) - Expect(err).NotTo(HaveOccurred()) - Expect(windows).To(HaveLen(2)) - Expect(windows[0].Start.AsTime().Unix()).To(Equal(startAlertWindow1.Unix())) - Expect(windows[0].End.AsTime().Unix()).To(Equal(endAlertWindow1.Unix())) - Expect(windows[1].Start.AsTime().Unix()).To(Equal(startAlertWindow2.Unix())) - Expect(windows[1].End.AsTime().Unix()).To(Equal(endAlertWindow2.Unix())) - }) - }) - - When("We receive SLO event data from the user", func() { - It("Should be able to convert the SLO event data to matching subsets for identical metric names", func() { - // total events empty ==> total events stays empty - goodEvents1 := []*sloapi.Event{ - { - Key: "code", - Vals: []string{"200"}, - }, - } - totalEvents1 := []*sloapi.Event{} - g, t := slo.ToMatchingSubsetIdenticalMetric(goodEvents1, totalEvents1) - Expect(g).To(Equal(goodEvents1)) - Expect(t).To(Equal(totalEvents1)) - - // management api server will pass in empty events to an empty list, - // because it hates us - totalEvents1 = []*sloapi.Event{{}} // in this case, this is what it looks like - g, t = slo.ToMatchingSubsetIdenticalMetric(goodEvents1, totalEvents1) - Expect(g).To(Equal(goodEvents1)) - Expect(t).To(Equal(totalEvents1)) - - // to be dilligent we should also check when t.Vals is nil - totalEvents1 = []*sloapi.Event{{Key: "code"}} - g, t = slo.ToMatchingSubsetIdenticalMetric(goodEvents1, totalEvents1) - Expect(g).To(Equal(goodEvents1)) - Expect(t).To(Equal(totalEvents1)) - - // should fill in missing subset - goodEvents2 := []*sloapi.Event{ - { - Key: "code", - Vals: []string{"200"}, - }, - } - totalEvents2 := []*sloapi.Event{ - { - Key: "code", - Vals: []string{"500", "503"}, - }, - } - goodEvents2Copy := make([]string, len(goodEvents2[0].Vals)) - copy(goodEvents2Copy, goodEvents2[0].Vals) - totalEvents2Copy := make([]string, len(totalEvents2[0].Vals)) - copy(totalEvents2Copy, totalEvents2[0].Vals) - g, t = slo.ToMatchingSubsetIdenticalMetric(goodEvents2, totalEvents2) - Expect(g).To(Equal(goodEvents2)) - Expect(t[0].Key).To(Equal(totalEvents2[0].Key)) - Expect(t[0].Vals).To(ConsistOf(append(goodEvents2Copy, totalEvents2Copy...))) - - // should coerce subsets with different filters - goodEvents3 := []*sloapi.Event{ - { - Key: "code", - Vals: []string{"200"}, - }, - } - totalEvents3 := []*sloapi.Event{ - { - Key: "handler", - Vals: []string{"/ready"}, - }, - } - g, t = slo.ToMatchingSubsetIdenticalMetric(goodEvents3, totalEvents3) - Expect(g).To(Equal([]*sloapi.Event{ - { - Key: "code", - Vals: []string{"200"}, - }, - { - Key: "handler", - Vals: []string{"/ready"}, - }, - })) - Expect(t).To(Equal(totalEvents3)) - - goodEvents4 := []*sloapi.Event{{ - Key: "code", - Vals: []string{"200"}}} - totalEvents4 := []*sloapi.Event{ - { - Key: "code", - Vals: []string{"500", "503"}, - }, - } - - g, t = slo.ToMatchingSubsetIdenticalMetric(goodEvents4, totalEvents4) - Expect(Expect(g[0].Key).To(Equal(goodEvents4[0].Key))) - Expect(g[0].Vals).To(ConsistOf([]string{"200"})) - Expect(t[0].Vals).To(ConsistOf(slo.LeftJoinSlice(goodEvents4[0].Vals, totalEvents4[0].Vals))) - - }) - - Specify("The event matching subset algorithm should be robust to chaos testing", func() { - // FIXME: this test is kinda hacky and poorly written - for numTests := 0; numTests < 15; numTests++ { - goodEventNum := rand.Intn(10) + 1 - goodEventValNum := rand.Intn(1000) + 1 - totalEventNum := rand.Intn(10) + 1 - - goodEvents := make([]*sloapi.Event, goodEventNum) - totalEvents := make([]*sloapi.Event, totalEventNum) - goodEventCache := make(map[string]struct{}) - goodEventValsCache := make(map[string]map[string]struct{}) - gen := namegenerator.NewNameGenerator(time.Now().UnixNano()) - for i := 0; i < goodEventNum; i++ { - name := gen.Generate() - vals := []string{} - for j := 0; j < goodEventValNum; j++ { - newVal := gen.Generate() - vals = append(vals, newVal) - if _, ok := goodEventValsCache[name]; !ok { - goodEventValsCache[name] = make(map[string]struct{}) - } - goodEventValsCache[name][newVal] = struct{}{} - } - goodEvents[i] = &sloapi.Event{ - Key: name, - Vals: vals, - } - goodEventCache[name] = struct{}{} - } - for i := 0; i < totalEventNum; i++ { - isMany := rand.Intn(2) - var totalEventValNum int - if isMany == 0 { - totalEventValNum = rand.Intn(1000) - } else { - totalEventValNum = rand.Intn(3) - } - - // 50% chance to use an existing key from goodEvents - var code string - var vals []string - if rand.Intn(2) == 0 && len(goodEventCache) > 0 { // use existing key - if goodEventNum < 0 { - panic(goodEvents) - } - index := rand.Intn(goodEventNum) - code = goodEvents[index].Key - delete(goodEventCache, code) - for j := 0; j < totalEventValNum; j++ { - //25% chance to use existing value from goodEvents - if rand.Intn(4) == 0 && len(goodEventValsCache[code]) > 0 { - keys := make([]string, len(goodEventValsCache[code])) - - x := 0 - for k := range goodEventValsCache[code] { - keys[x] = k - x++ - } - index := rand.Intn(len(keys)) - vals = append(vals, keys[index]) - delete(goodEventValsCache[code], keys[index]) - } else { - vals = append(vals, gen.Generate()) - } - } - } else { //create new key - name := gen.Generate() - code = name - for j := 0; j < totalEventValNum; j++ { - vals = append(vals, gen.Generate()) - } - } - totalEvents[i] = &sloapi.Event{ - Key: code, - Vals: vals, - } - } - invalid := false - for i := 0; i < len(totalEvents); i++ { - if totalEvents[i].Vals == nil || len(totalEvents[i].Vals) == 0 { - invalid = true - break - } - if totalEvents[i] == nil { - invalid = true - break - } - } - if invalid == true { - continue // FIXME: not sure why we are getting invalid constructions - } - newGoodEvents, newTotalEvents := slo.ToMatchingSubsetIdenticalMetric(goodEvents, totalEvents) - expectValidEventSubsets(newGoodEvents, newTotalEvents) - } - }) - }) - - When("We use SLO objects to construct rules", func() { - Specify("Label pairs should be able to construct promQL filters", func() { - fmt.Println(pPort) - goodLabelPairs := slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200"}, - }, - } - Expect(goodLabelPairs.Construct()).To(Equal(",code=~\"200\"")) - totalLabelPairs := slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200", "500", "503"}, - }, - } - Expect(totalLabelPairs.Construct()).To(Equal(",code=~\"200|500|503\"")) - }) - - It("Should construct an SLO object", func() { - sloObj := slo.NewSLO( - "slo-name", - "30d", - 99.99, - "prometheus", - slo.Metric("prometheus_http_requests_total"), - slo.Metric("prometheus_http_requests_total"), - map[string]string{ - "important": "true", - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200"}, - }, - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200", "500", "503"}, - }, - }, - ) - Expect(sloObj).ToNot(BeNil()) - Expect(sloObj.GetId()).ToNot(BeNil()) - sloObj2 := slo.SLOFromId( - - "slo-name", - "30d", - 99.99, - "prometheus", - "", - "slo-operator", - map[string]string{ - "important": "true", - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200"}, - }, - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200", "500", "503"}, - }, - }, - sloObj.GetId(), - ) - Expect(sloObj2).ToNot(BeNil()) - Expect(sloObj2.GetId()).To(Equal(sloObj.GetId())) - }) - Specify("SLO objects should be able to create valid SLI Prometheus rules", func() { - sloObj := slo.NewSLO( - "slo-name", - "30d", - 99.99, - "prometheus", - "prometheus_http_requests_total", - "prometheus_http_requests_total", - map[string]string{ - "important": "true", - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200"}, - }, - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200", "500", "503"}, - }, - }, - ) - query, err := sloObj.RawSLIQuery("5m") - Expect(err).To(Succeed()) - Expect(query).NotTo(Equal("")) - _, err = parser.ParseExpr(query) - Expect(err).To(Succeed()) - }) - - Specify("SLO objects should be able to create valid metadata Prometheus rules", func() { - sloObj := slo.NewSLO( - "slo-name", - "30d", - 99.99, - "prometheus", - "prometheus_http_requests_total", - "prometheus_http_requests_total", - map[string]string{ - "important": "true", - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200"}, - }, - }, - slo.LabelPairs{ - { - Key: "code", - Vals: []string{"200", "500", "503"}, - }, - }, - ) - dash := sloObj.RawDashboardInfoQuery() - _, err := parser.ParseExpr(dash) - Expect(err).To(Succeed()) - rawBudget := sloObj.RawErrorBudgetQuery() - _, err = parser.ParseExpr(rawBudget) - Expect(err).To(Succeed()) - - rawObjective := sloObj.RawObjectiveQuery() - _, err = parser.ParseExpr(rawObjective) - Expect(err).To(Succeed()) - rawRemainingBudget := sloObj.RawBudgetRemainingQuery() - _, err = parser.ParseExpr(rawRemainingBudget) - Expect(err).To(Succeed()) - rawPeriodAsVectorDays := sloObj.RawPeriodDurationQuery() - _, err = parser.ParseExpr(rawPeriodAsVectorDays) - Expect(err).To(Succeed()) - // burn rate - curBurnRate := sloObj.RawCurrentBurnRateQuery() - _, err = parser.ParseExpr(curBurnRate) - Expect(err).To(Succeed()) - periodBurnRate := sloObj.RawPeriodBurnRateQuery() - _, err = parser.ParseExpr(periodBurnRate) - Expect(err).To(Succeed()) - }) - - Specify("SLO objects should be able to create valid alerting Prometheus rules", func() { - interval := time.Second - ralerts := sloObj.ConstructAlertingRuleGroup(&interval) - for _, alertRule := range ralerts.Rules { - _, err := parser.ParseExpr(alertRule.Expr.Value) - Expect(err).To(Succeed()) - } - }) - - Specify("SLO objects should be able to create valid Cortex recording rule groups", func() { - interval := time.Second - rrecording := sloObj.ConstructRecordingRuleGroup(&interval) - out, err := yaml.Marshal(rrecording) - Expect(err).To(Succeed()) - y := rulefmt.RuleGroup{} - err = yaml.Unmarshal(out, &y) - Expect(err).To(Succeed()) - //FIXME: when joe is back, should add pkg/rules to his fork of cortex-tools - //errors := rules.ValidateRuleGroup(y) - //Expect(errors).To(BeEmpty()) - - }) - - Specify("SLO objects should be able to create valid Cortex metadata rule groups", func() { - interval := time.Second - rmetadata := sloObj.ConstructMetadataRules(&interval) - out, err := yaml.Marshal(rmetadata) - Expect(err).To(Succeed()) - y := rulefmt.RuleGroup{} - err = yaml.Unmarshal(out, &y) - Expect(err).To(Succeed()) - //FIXME: when joe is back, should add pkg/rules to his fork of cortex-tools - //errors := rules.ValidateRuleGroup(y) - //Expect(errors).To(BeEmpty()) - - }) - - Specify("SLO objects should be able to create valid Cortex alerting rule groups", func() { - interval := time.Second - ralerts := sloObj.ConstructMetadataRules(&interval) - out, err := yaml.Marshal(ralerts) - Expect(err).To(Succeed()) - y := rulefmt.RuleGroup{} - err = yaml.Unmarshal(out, &y) - Expect(err).To(Succeed()) - //FIXME: when joe is back, should add pkg/rules to his fork of cortex-tools - //errors := rules.ValidateRuleGroup(y) - //Expect(errors).To(BeEmpty()) - }) - }) - - When("When use raw SLO constructions with cortex admin client", func() { - Specify("The individual parts of the raw SLI queries should return data from cortex", func() { - // needed to ensure that prometheus agent registers a 200 status code - // otherwise the test is flaky - rawGood, err := sloObj.RawGoodEventsQuery("5m") - Expect(err).NotTo(HaveOccurred()) - rawTotal, err := sloObj.RawTotalEventsQuery("5m") - Expect(err).NotTo(HaveOccurred()) - Eventually(func() error { - return nil - }, time.Second*90, time.Second*30).Should(BeNil()) - respGood, err := adminClient.Query(env.Context(), &cortexadmin.QueryRequest{ - Query: rawGood, - Tenants: []string{"agent"}, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(respGood.Data).NotTo(BeEmpty()) - qres, err := compat.UnmarshalPrometheusResponse(respGood.Data) - Expect(err).NotTo(HaveOccurred()) - goodVector, err := qres.GetVector() - Expect(err).NotTo(HaveOccurred()) - Expect(goodVector).NotTo(BeNil()) - Expect(*goodVector).NotTo(BeEmpty()) - for _, sample := range *goodVector { - Expect(sample.Value).To(BeNumerically(">=", 0)) - Expect(sample.Timestamp).To(BeNumerically(">=", 0)) - } - - respTotal, err := adminClient.Query(env.Context(), &cortexadmin.QueryRequest{ - Query: rawTotal, - Tenants: []string{"agent"}, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(respTotal.Data).NotTo(BeEmpty()) - qresTotal, err := compat.UnmarshalPrometheusResponse(respTotal.Data) - Expect(err).NotTo(HaveOccurred()) - totalVector, err := qresTotal.GetVector() - Expect(err).NotTo(HaveOccurred()) - Expect(totalVector).NotTo(BeNil()) - Expect(*totalVector).NotTo(BeEmpty()) - for _, sample := range *totalVector { - Expect(sample.Value).To(BeNumerically(">=", 0)) - Expect(sample.Timestamp).To(BeNumerically(">=", 0)) - } - - adHocSliErrorRatioQuery := fmt.Sprintf(" 1 - (%s)/(%s)", rawGood, rawTotal) - respSli, err := adminClient.Query(env.Context(), &cortexadmin.QueryRequest{ - Tenants: []string{"agent"}, - Query: adHocSliErrorRatioQuery, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(respSli.Data).NotTo(BeEmpty()) - qresSli, err := compat.UnmarshalPrometheusResponse(respSli.Data) - Expect(err).NotTo(HaveOccurred()) - sliErrorRatioVector, err := qresSli.GetVector() - Expect(err).NotTo(HaveOccurred()) - Expect(sliErrorRatioVector).NotTo(BeNil()) - Expect(*sliErrorRatioVector).NotTo(BeEmpty()) - for _, sample := range *sliErrorRatioVector { - Expect(sample.Value).To(BeNumerically(">=", 0)) - Expect(sample.Value).To(BeNumerically("<=", 1)) - Expect(sample.Timestamp).To(BeNumerically(">=", 0)) - } - }) - - Specify("All of the raw SLI queries should return data from cortex", func() { - time.Sleep(time.Second * 10) - interval := time.Second - rrecording := sloObj.ConstructRecordingRuleGroup(&interval) - for _, rawRule := range rrecording.Rules { - resp, err := adminClient.Query(env.Context(), &cortexadmin.QueryRequest{ - Query: rawRule.Expr.Value, - Tenants: []string{"agent"}, - }) - Expect(err).NotTo(HaveOccurred()) - rawBytes := resp.Data - qres, err := compat.UnmarshalPrometheusResponse(rawBytes) - Expect(err).NotTo(HaveOccurred()) - Expect(qres).NotTo(BeNil()) - recordingVector, err := qres.GetVector() - Expect(err).NotTo(HaveOccurred()) - Expect(recordingVector).NotTo(BeNil()) - Expect(*recordingVector).NotTo(BeEmpty()) - for _, sample := range *recordingVector { - Expect(sample.Value).To(BeNumerically(">=", 0)) - Expect(sample.Timestamp).To(BeNumerically(">=", 0)) - } - } - }) - - Specify("All of the raw Metadata queries should return data from cortex", func() { - // TODO : need to replace all rule names in each of these Expr's to their real expr - }) - - Specify("All of the raw alert queries should return data from cortex", func() { - // TODO: need to replace all rule names in each of these Expr's to their real expr - }) - - Specify("After aplying the rules to cortex, each rule name should evaluate to non-empty data", func() { - // Manually apply the SLI recording rules - interval := time.Second - rrecording := sloObj.ConstructRecordingRuleGroup(&interval) - rmetadata := sloObj.ConstructMetadataRules(&interval) - ralerts := sloObj.ConstructAlertingRuleGroup(&interval) - - outRecording, err := yaml.Marshal(rrecording) - Expect(err).To(Succeed()) - - _, err = adminClient.LoadRules(env.Context(), &cortexadmin.LoadRuleRequest{ - ClusterId: "agent", - Namespace: "slo", - YamlContent: outRecording, - }) - Expect(err).NotTo(HaveOccurred()) - outMetadata, err := yaml.Marshal(rmetadata) - Expect(err).NotTo(HaveOccurred()) - _, err = adminClient.LoadRules(env.Context(), &cortexadmin.LoadRuleRequest{ - ClusterId: "agent", - Namespace: "slo", - YamlContent: outMetadata, - }) - Expect(err).NotTo(HaveOccurred()) - outAlerts, err := yaml.Marshal(ralerts) - Expect(err).NotTo(HaveOccurred()) - _, err = adminClient.LoadRules(env.Context(), &cortexadmin.LoadRuleRequest{ - ClusterId: "agent", - Namespace: "slo", - YamlContent: outAlerts, - }) - Expect(err).NotTo(HaveOccurred()) - - Eventually(func() error { - resp, err := adminClient.ListRules(env.Context(), &cortexadmin.ListRulesRequest{ - ClusterId: []string{"agent"}, - }) - if err != nil { - return err - } - for _, group := range resp.Data.Groups { - if strings.Contains(group.Name, sloObj.GetId()) { - return nil - } - } - return nil - }, time.Minute*2, time.Second*30).Should(Succeed()) - - // check the recording rule names to make sure they return data - Eventually(func() error { - for _, rawRule := range rrecording.Rules { - ruleVector, err := slo.QuerySLOComponentByRawQuery(env.Context(), adminClient, rawRule.Expr.Value, "agent") - if err != nil { - return err - } - - if ruleVector == nil || len(*ruleVector) == 0 { - return fmt.Errorf("expect rule vector to contain data") - } - for _, sample := range *ruleVector { - if sample.Timestamp == 0 { - return fmt.Errorf("expect sample.Timestamp to contain data") - } - } - } - return nil - }, time.Minute*2, time.Second*30).Should(Succeed()) - - Eventually(func() error { - for _, rawRule := range rmetadata.Rules { - ruleVector, err := slo.QuerySLOComponentByRecordName(env.Context(), adminClient, rawRule.Record.Value, "agent") - if err != nil { - return err - } - - if ruleVector == nil || len(*ruleVector) == 0 { - return fmt.Errorf("expect rule vector to contain data") - } - for _, sample := range *ruleVector { - if sample.Timestamp == 0 { - return fmt.Errorf("expect sample.Timestamp to contain data") - } - } - } - return nil - }, time.Minute*2, time.Second*30).Should(Succeed()) - - Eventually(func() error { - rawSevereAlertQuery, rawCriticalAlertQuery := sloObj.ConstructRawAlertQueries() - // @debug - //rawSevereAlertQueryComponents := strings.Split(rawSevereAlertQuery, "or") - //rawCriticalAlertQueryComponents := strings.Split(rawCriticalAlertQuery, "or") - //var alertRules []string - //alertRules = append(alertRules, rawSevereAlertQueryComponents...) - //alertRules = append(alertRules, rawCriticalAlertQueryComponents...) - alertRules := []string{rawSevereAlertQuery.Value, rawCriticalAlertQuery.Value} - for _, rawRule := range alertRules { - ruleVector, err := slo.QuerySLOComponentByRawQuery(env.Context(), adminClient, rawRule, "agent") - if err != nil { - return err - } - if ruleVector == nil || len(*ruleVector) == 0 { - return fmt.Errorf("expect rule vector to contain data") - } - for _, sample := range *ruleVector { - if sample.Timestamp == 0 { - return fmt.Errorf("expect sample.Timestamp to contain data") - } - } - } - return nil - }, time.Minute*2, time.Second*30).Should(Succeed()) - }) - }) -}) diff --git a/plugins/slo/pkg/slo/system.go b/plugins/slo/pkg/slo/system.go deleted file mode 100644 index 8b8a74a0df..0000000000 --- a/plugins/slo/pkg/slo/system.go +++ /dev/null @@ -1,48 +0,0 @@ -package slo - -import ( - "os" - - alertingv1 "github.com/rancher/opni/pkg/apis/alerting/v1" - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - "github.com/rancher/opni/pkg/plugins/apis/system" - "github.com/rancher/opni/pkg/slo/shared" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" -) - -func (p *Plugin) UseManagementAPI(client managementv1.ManagementClient) { - p.mgmtClient.Set(client) - <-p.ctx.Done() -} - -func (p *Plugin) UseKeyValueStore(client system.KeyValueStoreClient) { - p.storage.Set(StorageAPIs{ - SLOs: system.NewKVStoreClient[*sloapi.SLOData](client), - Services: system.NewKVStoreClient[*sloapi.Service](client), - Metrics: system.NewKVStoreClient[*sloapi.Metric](client), - }) - <-p.ctx.Done() -} - -func (p *Plugin) UseAPIExtensions(intf system.ExtensionClientInterface) { - cc, err := intf.GetClientConn(p.ctx, "CortexAdmin", "AlertEndpoints") - if err != nil { - p.logger.Error("failed to get cortex admin client", "error", err) - if p.ctx.Err() != nil { - // Plugin is shutting down, don't exit - return - } - os.Exit(1) - } - adminClient := cortexadmin.NewCortexAdminClient(cc) - alertingEndpointClient := alertingv1.NewAlertEndpointsClient(cc) - - p.adminClient.Set(adminClient) - p.alertEndpointClient.Set(alertingEndpointClient) - RegisterDatasource( - shared.MonitoringDatasource, - NewSLOMonitoringStore(p, p.logger), - NewMonitoringServiceBackend(p, p.logger), - ) -} diff --git a/plugins/slo/pkg/slo/util.go b/plugins/slo/pkg/slo/util.go deleted file mode 100644 index 31c7545cbb..0000000000 --- a/plugins/slo/pkg/slo/util.go +++ /dev/null @@ -1,68 +0,0 @@ -package slo - -import ( - "text/template" - "time" - - prommodel "github.com/prometheus/common/model" -) - -const ( - tplKeyWindow = "window" -) - -// Reference : https://github.com/slok/sloth/blob/eddd8145a696c3dc6d423e9d50cdb906186a52a3/internal/prometheus/recording_rules.go#L308 -var burnRateRecordingExprTpl = template.Must(template.New("burnRateExpr").Option("missingkey=error").Parse(`{{ .SLIErrorMetric }}{{ .MetricFilter }} -/ on({{ .SLOIDName }}, {{ .SLOLabelName }}, {{ .SLOServiceName }}) group_left -{{ .ErrorBudgetRatioMetric }}{{ .MetricFilter }} -`)) - -// Reference : https://github.com/slok/sloth/blob/2de193572284e36189fe78ab33beb7e2b339b0f8/internal/prometheus/alert_rules.go#L109 -// Multiburn multiwindow alert template. -var mwmbAlertTplBool = template.Must(template.New("mwmbAlertTpl").Option("missingkey=error").Parse(`(max({{ .QuickShortMetric }}{{ .MetricFilter}} > bool ({{ .QuickShortBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}) and max({{ .QuickLongMetric }}{{ .MetricFilter}} > bool ({{ .QuickLongBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }})) or (max({{ .SlowShortMetric }}{{ .MetricFilter }} > bool ({{ .SlowShortBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}) and max({{ .SlowQuickMetric }}{{ .MetricFilter }} > bool ({{ .SlowQuickBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}))`)) -var mwmbAlertTpl = template.Must(template.New("mwmbAlertTpl").Option("missingkey=error").Parse(`(max({{ .QuickShortMetric }}{{ .MetricFilter}} > ({{ .QuickShortBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}) and max({{ .QuickLongMetric }}{{ .MetricFilter}} > ({{ .QuickLongBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }})) or (max({{ .SlowShortMetric }}{{ .MetricFilter }} > ({{ .SlowShortBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}) and max({{ .SlowQuickMetric }}{{ .MetricFilter }} > ({{ .SlowQuickBurnFactor }} * {{ .ErrorBudgetRatio }})) without ({{ .WindowLabel }}))`)) - -// Pretty simple durations for prometheus. -func TimeDurationToPromStr(t time.Duration) string { - return prommodel.Duration(t).String() -} - -func MergeLabels(ms ...map[string]string) map[string]string { - res := map[string]string{} - for _, m := range ms { - for k, v := range m { - res[k] = v - } - } - return res -} - -func LeftJoinSlice[T comparable](arr1, arr2 []T) []T { - result := make([]T, len(arr1)) - cache := map[T]struct{}{} - for i, v := range arr1 { - cache[v] = struct{}{} - result[i] = v - } - for _, v := range arr2 { - if _, ok := cache[v]; !ok { - result = append(result, v) - } - } - return result -} - -func LeftJoinSliceAbstract[T any, S comparable](arr1, arr2 []T, getId func(T) S) []T { - result := make([]T, len(arr1)) - cache := map[S]struct{}{} - for i, v := range arr1 { - cache[getId(v)] = struct{}{} - result[i] = v - } - for _, v := range arr2 { - if _, ok := cache[getId(v)]; !ok { - result = append(result, v) - } - } - return result -} diff --git a/plugins/slo/pkg/slo/window.go b/plugins/slo/pkg/slo/window.go deleted file mode 100644 index 53e77132fa..0000000000 --- a/plugins/slo/pkg/slo/window.go +++ /dev/null @@ -1,172 +0,0 @@ -// Reference : https://github.dev/slok/sloth/tree/main/internal/alert/window.go -package slo - -import ( - "fmt" - "time" -) - -type Windows struct { - SLOPeriod time.Duration - PageQuick Window - PageSlow Window - TicketQuick Window - TicketSlow Window -} - -// https://sre.google/workbook/alerting-on-slos/ -// -// budgeting interval is the shortest interval to monitor in a window -func GenerateGoogleWindows(budgetingInterval time.Duration) *Windows { - pageQuickShort := budgetingInterval - pageQuickLong := budgetingInterval * 12 - - pageSlowShort := budgetingInterval * 6 - pageSlowLong := pageSlowShort * 12 - - ticketQuickShort := pageSlowShort * 4 - ticketQuickLong := ticketQuickShort * 12 - - ticketSlowShort := ticketQuickShort * 3 - ticketSlowLong := ticketQuickLong * 3 - - return &Windows{ - SLOPeriod: budgetingInterval, - PageQuick: Window{ - LongWindow: pageQuickLong, - ShortWindow: pageQuickShort, - ErrorBudgetPercent: 2, - }, - PageSlow: Window{ - LongWindow: pageSlowLong, - ShortWindow: pageSlowShort, - ErrorBudgetPercent: 5, - }, - TicketQuick: Window{ - LongWindow: ticketQuickLong, - ShortWindow: ticketQuickShort, - ErrorBudgetPercent: 10, - }, - TicketSlow: Window{ - LongWindow: ticketSlowLong, - ShortWindow: ticketSlowShort, - ErrorBudgetPercent: 10, - }, - } -} - -// https://sre.google/workbook/alerting-on-slos/ -func WindowDefaults(period time.Duration) *Windows { - return &Windows{ - SLOPeriod: period, - PageQuick: Window{ - LongWindow: time.Minute * 60, - ShortWindow: time.Minute * 5, - ErrorBudgetPercent: 2, - }, - PageSlow: Window{ - LongWindow: time.Hour * 6, - ShortWindow: time.Minute * 30, - ErrorBudgetPercent: 5, - }, - TicketQuick: Window{ - LongWindow: (time.Hour * 24) * 3, - ShortWindow: time.Hour * 6, - ErrorBudgetPercent: 10, - }, - TicketSlow: Window{ - LongWindow: (time.Hour * 24) * 3, - ShortWindow: time.Hour * 6, - ErrorBudgetPercent: 10, - }, - } -} - -type Window struct { - // ErrorBudgetPercent is the error budget % consumed for a full time window. - // Google gives us some defaults in its SRE workbook that work correctly most of the times: - // - Page quick: 2% - // - Page slow: 5% - // - Ticket quick: 10% - // - Ticket slow: 10% - ErrorBudgetPercent float64 - // ShortWindow is the small window used on the alerting part to stop alerting - // during a long window because we consumed a lot of error budget but the problem - // is already gone. - ShortWindow time.Duration - // LongWindow is the long window used to alert based on the errors happened on that - // long window. - LongWindow time.Duration -} - -func (w Window) Validate() error { - if w.LongWindow == 0 { - return fmt.Errorf("long window is required") - } - - if w.ShortWindow == 0 { - return fmt.Errorf("short window is required") - } - - if w.ErrorBudgetPercent == 0 { - return fmt.Errorf("error budget is required") - } - - return nil -} - -func (w Windows) Validate() error { - if w.SLOPeriod == 0 { - return fmt.Errorf("slo period is required") - } - - err := w.PageQuick.Validate() - if err != nil { - return fmt.Errorf("invalid page quick: %w", err) - } - - err = w.PageSlow.Validate() - if err != nil { - return fmt.Errorf("invalid page slow: %w", err) - } - - err = w.TicketQuick.Validate() - if err != nil { - return fmt.Errorf("invalid ticket quick: %w", err) - } - - err = w.TicketSlow.Validate() - if err != nil { - return fmt.Errorf("invalid ticket slow: %w", err) - } - - return nil -} - -// Error budget speeds based on a full time window, however once we have the factor (speed) -// the value can be used with any time window. -func (w Windows) GetSpeedPageQuick() float64 { - return w.GetBurnRateFactor(w.SLOPeriod, float64(w.PageQuick.ErrorBudgetPercent), w.PageQuick.LongWindow) -} -func (w Windows) GetSpeedPageSlow() float64 { - return w.GetBurnRateFactor(w.SLOPeriod, float64(w.PageSlow.ErrorBudgetPercent), w.PageSlow.LongWindow) -} -func (w Windows) GetSpeedTicketQuick() float64 { - return w.GetBurnRateFactor(w.SLOPeriod, float64(w.TicketQuick.ErrorBudgetPercent), w.TicketQuick.LongWindow) -} -func (w Windows) GetSpeedTicketSlow() float64 { - return w.GetBurnRateFactor(w.SLOPeriod, float64(w.TicketSlow.ErrorBudgetPercent), w.TicketSlow.LongWindow) -} - -// getBurnRateFactor calculates the burnRateFactor (speed) needed to consume all the error budget available percent -// in a specific time window taking into account the total time window. -func (w Windows) GetBurnRateFactor(totalWindow time.Duration, errorBudgetPercent float64, consumptionWindow time.Duration) float64 { - // First get the total hours required to consume the % of the error budget in the total window. - hoursRequiredConsumption := errorBudgetPercent * totalWindow.Hours() / 100 - - // Now calculate how much is the factor required for the hours consumption, in case we would need to use - // a different time window (e.g: hours required: 36h, if we want to do it in 6h: would be `x6`). - speed := hoursRequiredConsumption / consumptionWindow.Hours() - - return speed -} diff --git a/plugins/slo/test/test_plugin.go b/plugins/slo/test/test_plugin.go deleted file mode 100644 index 9eed321f90..0000000000 --- a/plugins/slo/test/test_plugin.go +++ /dev/null @@ -1,15 +0,0 @@ -package test - -import ( - "time" - - "github.com/rancher/opni/pkg/plugins/meta" - "github.com/rancher/opni/pkg/test" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "github.com/rancher/opni/plugins/slo/pkg/slo" -) - -func init() { - sloapi.MinEvaluateInterval = time.Second - test.EnablePlugin(meta.ModeGateway, slo.Scheme) -} diff --git a/test/integration/managementapi_cert_test.go b/test/integration/managementapi_cert_test.go deleted file mode 100644 index c06cce5bd3..0000000000 --- a/test/integration/managementapi_cert_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package integration_test - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "google.golang.org/protobuf/types/known/emptypb" - - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - "github.com/rancher/opni/pkg/test" - _ "github.com/rancher/opni/plugins/example/test" -) - -//#region Test Setup - -var _ = Describe("Management API Cerificate Management Tests", Ordered, Label("integration"), func() { - var environment *test.Environment - var client managementv1.ManagementClient - BeforeAll(func() { - environment = &test.Environment{} - Expect(environment.Start()).To(Succeed()) - DeferCleanup(environment.Stop) - client = environment.NewManagementClient() - }) - - //#endregion - - //#region Happy Path Tests - - It("can retrieve full certification chain information", func() { - certsInfo, err := client.CertsInfo(context.Background(), &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - - leaf := certsInfo.Chain[0] - root := certsInfo.Chain[len(certsInfo.Chain)-1] - - Expect(root.Issuer).To(Equal("CN=Example Root CA")) - Expect(root.Subject).To(Equal("CN=Example Root CA")) - Expect(root.IsCA).To(BeTrue()) - Expect(root.Fingerprint).NotTo(BeEmpty()) - - Expect(leaf.Issuer).To(Equal("CN=Example Root CA")) - Expect(leaf.Subject).To(Equal("CN=leaf")) - Expect(leaf.IsCA).To(BeFalse()) - Expect(leaf.Fingerprint).NotTo(BeEmpty()) - }) - - //#endregion - - //#region Edge Case Tests - - //TODO: Add Certificate Edge Case Tests - - //#endregion -}) diff --git a/test/plugins/alerting/alerting_suite_test.go b/test/plugins/alerting/alerting_suite_test.go index 4c29cdc187..acab5e22be 100644 --- a/test/plugins/alerting/alerting_suite_test.go +++ b/test/plugins/alerting/alerting_suite_test.go @@ -8,9 +8,7 @@ import ( . "github.com/onsi/gomega" "github.com/prometheus/common/model" "github.com/rancher/opni/pkg/alerting/drivers/routing" - "github.com/rancher/opni/pkg/test" _ "github.com/rancher/opni/pkg/test/setup" - "github.com/rancher/opni/pkg/test/testruntime" _ "github.com/rancher/opni/plugins/alerting/test" _ "github.com/rancher/opni/plugins/metrics/test" "github.com/samber/lo" @@ -40,16 +38,16 @@ func TestAlerting(t *testing.T) { RunSpecs(t, "Alerting Suite") } -var env *test.Environment -var tmpConfigDir string +// var env *test.Environment +// var tmpConfigDir string -var _ = BeforeSuite(func() { - testruntime.IfIntegration(func() { - env = &test.Environment{} - Expect(env).NotTo(BeNil()) - Expect(env.Start()).To(Succeed()) - DeferCleanup(env.Stop) - tmpConfigDir = env.GenerateNewTempDirectory("alertmanager-config") - Expect(tmpConfigDir).NotTo(Equal("")) - }) -}) +// var _ = BeforeSuite(func() { +// testruntime.IfIntegration(func() { +// env = &test.Environment{} +// Expect(env).NotTo(BeNil()) +// Expect(env.Start()).To(Succeed()) +// DeferCleanup(env.Stop) +// tmpConfigDir = env.GenerateNewTempDirectory("alertmanager-config") +// Expect(tmpConfigDir).NotTo(Equal("")) +// }) +// }) diff --git a/test/plugins/alerting/alerting_test.go b/test/plugins/alerting/alerting_test.go index eef4aee75c..58f85541e8 100644 --- a/test/plugins/alerting/alerting_test.go +++ b/test/plugins/alerting/alerting_test.go @@ -22,7 +22,6 @@ import ( managementv1 "github.com/rancher/opni/pkg/apis/management/v1" "github.com/rancher/opni/pkg/test" "github.com/rancher/opni/pkg/test/alerting" - "github.com/rancher/opni/pkg/test/testruntime" "github.com/rancher/opni/plugins/alerting/apis/alertops" "github.com/samber/lo" "golang.org/x/exp/slices" @@ -39,51 +38,6 @@ const ( numNotificationServers = 2 ) -func init() { - testruntime.IfIntegration(func() { - BuildAlertingClusterIntegrationTests([]*alertops.ClusterConfiguration{ - // { - // NumReplicas: 1, - // ResourceLimits: &alertops.ResourceLimitSpec{ - // Cpu: "100m", - // Memory: "100Mi", - // Storage: "5Gi", - // }, - // ClusterSettleTimeout: "1m", - // ClusterGossipInterval: "1m", - // ClusterPushPullInterval: "1m", - // }, - { - NumReplicas: 3, - ResourceLimits: &alertops.ResourceLimitSpec{ - Cpu: "100m", - Memory: "100Mi", - Storage: "5Gi", - }, - ClusterSettleTimeout: "1m", - ClusterGossipInterval: "1m", - ClusterPushPullInterval: "1m", - }, - }, - func() alertops.AlertingAdminClient { - return alertops.NewAlertingAdminClient(env.ManagementClientConn()) - }, - func() alertingv1.AlertConditionsClient { - return env.NewAlertConditionsClient() - }, - func() alertingv1.AlertEndpointsClient { - return env.NewAlertEndpointsClient() - }, - func() alertingv1.AlertNotificationsClient { - return env.NewAlertNotificationsClient() - }, - func() managementv1.ManagementClient { - return env.NewManagementClient() - }, - ) - }) -} - type agentWithContext struct { id string context.Context @@ -91,829 +45,843 @@ type agentWithContext struct { port int } -func BuildAlertingClusterIntegrationTests( - clusterConfigurations []*alertops.ClusterConfiguration, - alertingAdminConstructor func() alertops.AlertingAdminClient, - alertingConditionsConstructor func() alertingv1.AlertConditionsClient, - alertingEndpointsConstructor func() alertingv1.AlertEndpointsClient, - alertingNotificationsConstructor func() alertingv1.AlertNotificationsClient, - mgmtClientConstructor func() managementv1.ManagementClient, -) bool { - return Describe("Alerting Cluster Integration tests", Ordered, Label("integration"), func() { - var alertClusterClient alertops.AlertingAdminClient - var alertEndpointsClient alertingv1.AlertEndpointsClient - var alertConditionsClient alertingv1.AlertConditionsClient - var alertNotificationsClient alertingv1.AlertNotificationsClient - var mgmtClient managementv1.ManagementClient - var numAgents int - - // contains agent id and other useful metadata and functions - var agents []*agentWithContext - // physical servers that receive opni alerting notifications - var servers []*alerting.MockIntegrationWebhookServer - // physical servers that receive all opni alerting notifications - var notificationServers []*alerting.MockIntegrationWebhookServer - // conditionsIds => endopintIds - expectedRouting := map[string]*alertingv1.ConditionReferenceList{} - // maps condition ids where agents are disconnect to their webhook ids - involvedDisconnects := map[string][]string{} - - var httpProxyClient *http.Client - When("Installing the Alerting Cluster", func() { - BeforeAll(func() { - alertClusterClient = alertingAdminConstructor() - alertEndpointsClient = alertingEndpointsConstructor() - alertConditionsClient = alertingConditionsConstructor() - alertNotificationsClient = alertingNotificationsConstructor() - mgmtClient = mgmtClientConstructor() - numAgents = 5 - tlsConfig := env.GatewayClientTLSConfig() - httpProxyClient = &http.Client{ - Transport: &http.Transport{ - DialTLS: func(network, addr string) (net.Conn, error) { - conn, err := tls.Dial(network, addr, tlsConfig) - return conn, err - }, +var _ = Describe("Alerting Cluster Integration tests", Ordered, Label("integration"), func() { + var env *test.Environment + + var clusterConfigurations = []*alertops.ClusterConfiguration{ + { + NumReplicas: 3, + ResourceLimits: &alertops.ResourceLimitSpec{ + Cpu: "100m", + Memory: "100Mi", + Storage: "5Gi", + }, + ClusterSettleTimeout: "1m", + ClusterGossipInterval: "1m", + ClusterPushPullInterval: "1m", + }, + } + + var alertClusterClient alertops.AlertingAdminClient + var alertEndpointsClient alertingv1.AlertEndpointsClient + var alertConditionsClient alertingv1.AlertConditionsClient + var alertNotificationsClient alertingv1.AlertNotificationsClient + var mgmtClient managementv1.ManagementClient + var numAgents int + + // contains agent id and other useful metadata and functions + var agents []*agentWithContext + // physical servers that receive opni alerting notifications + var servers []*alerting.MockIntegrationWebhookServer + // physical servers that receive all opni alerting notifications + var notificationServers []*alerting.MockIntegrationWebhookServer + // conditionsIds => endopintIds + expectedRouting := map[string]*alertingv1.ConditionReferenceList{} + // maps condition ids where agents are disconnect to their webhook ids + involvedDisconnects := map[string][]string{} + + var httpProxyClient *http.Client + When("Installing the Alerting Cluster", func() { + BeforeAll(func() { + env = &test.Environment{} + Expect(env).NotTo(BeNil()) + Expect(env.Start()).To(Succeed()) + DeferCleanup(env.Stop) + tmpConfigDir := env.GenerateNewTempDirectory("alertmanager-config") + Expect(tmpConfigDir).NotTo(Equal("")) + + alertClusterClient = alertops.NewAlertingAdminClient(env.ManagementClientConn()) + alertEndpointsClient = env.NewAlertEndpointsClient() + alertConditionsClient = env.NewAlertConditionsClient() + alertNotificationsClient = env.NewAlertNotificationsClient() + mgmtClient = env.NewManagementClient() + numAgents = 5 + tlsConfig := env.GatewayClientTLSConfig() + httpProxyClient = &http.Client{ + Transport: &http.Transport{ + DialTLS: func(network, addr string) (net.Conn, error) { + conn, err := tls.Dial(network, addr, tlsConfig) + return conn, err }, - } - }) + }, + } + }) - AfterEach(func() { - status, err := alertClusterClient.GetClusterStatus(env.Context(), &emptypb.Empty{}) + AfterEach(func() { + status, err := alertClusterClient.GetClusterStatus(env.Context(), &emptypb.Empty{}) + Expect(err).To(BeNil()) + installed := status.State != alertops.InstallState_NotInstalled && status.State != alertops.InstallState_Uninstalling + if installed { + info, err := alertClusterClient.Info(env.Context(), &emptypb.Empty{}) Expect(err).To(BeNil()) - installed := status.State != alertops.InstallState_NotInstalled && status.State != alertops.InstallState_Uninstalling - if installed { - info, err := alertClusterClient.Info(env.Context(), &emptypb.Empty{}) - Expect(err).To(BeNil()) - Expect(info).NotTo(BeNil()) - Expect(info.CurSyncId).NotTo(BeEmpty()) - ts := timestamppb.Now() - for _, comp := range info.Components { - Expect(comp.LastHandshake.AsTime()).To(BeTemporally("<", ts.AsTime())) - if comp.ConnectInfo.GetState() != alertops.SyncState_Synced { - if comp.ConnectInfo.SyncId != info.CurSyncId { - Expect(comp.LastHandshake.AsTime()).To(BeTemporally(">", ts.AsTime().Add(15*(-time.Second))), "sync is stuttering") - } + Expect(info).NotTo(BeNil()) + Expect(info.CurSyncId).NotTo(BeEmpty()) + ts := timestamppb.Now() + for _, comp := range info.Components { + Expect(comp.LastHandshake.AsTime()).To(BeTemporally("<", ts.AsTime())) + if comp.ConnectInfo.GetState() != alertops.SyncState_Synced { + if comp.ConnectInfo.SyncId != info.CurSyncId { + Expect(comp.LastHandshake.AsTime()).To(BeTemporally(">", ts.AsTime().Add(15*(-time.Second))), "sync is stuttering") } } } - }) - - for _, clusterConf := range clusterConfigurations { - It("should install the alerting cluster", func() { - _, err := alertClusterClient.InstallCluster(env.Context(), &emptypb.Empty{}) - Expect(err).To(BeNil()) + } + }) - Eventually(func() error { - status, err := alertClusterClient.GetClusterStatus(env.Context(), &emptypb.Empty{}) - if err != nil { - return err - } - if status.State != alertops.InstallState_Installed { - return fmt.Errorf("alerting cluster install state is %s", status.State.String()) - } - return nil - }, time.Second*5, time.Millisecond*200).Should(Succeed()) - }) + for _, clusterConf := range clusterConfigurations { + It("should install the alerting cluster", func() { + _, err := alertClusterClient.InstallCluster(env.Context(), &emptypb.Empty{}) + Expect(err).To(BeNil()) - It("should apply the configuration", func() { - _, err := alertClusterClient.ConfigureCluster(env.Context(), clusterConf) + Eventually(func() error { + status, err := alertClusterClient.GetClusterStatus(env.Context(), &emptypb.Empty{}) if err != nil { - if s, ok := status.FromError(err); ok { // conflict is ok if using default config - Expect(s.Code()).To(Equal(codes.FailedPrecondition)) - } + return err } - Expect(err).To(BeNil()) - Eventually(func() error { - status, err := alertClusterClient.GetClusterStatus(env.Context(), &emptypb.Empty{}) - if err != nil { - return err - } - if status.State != alertops.InstallState_Installed { - return fmt.Errorf("alerting cluster install state is %s", status.State.String()) - } - return nil - }, time.Second*5, time.Millisecond*200).Should(Succeed()) + if status.State != alertops.InstallState_Installed { + return fmt.Errorf("alerting cluster install state is %s", status.State.String()) + } + return nil + }, time.Second*5, time.Millisecond*200).Should(Succeed()) + }) - Eventually(func() error { - getConf, err := alertClusterClient.GetClusterConfiguration(env.Context(), &emptypb.Empty{}) - if !proto.Equal(getConf, clusterConf) { - return fmt.Errorf("cluster config not equal : not applied") - } + It("should apply the configuration", func() { + _, err := alertClusterClient.ConfigureCluster(env.Context(), clusterConf) + if err != nil { + if s, ok := status.FromError(err); ok { // conflict is ok if using default config + Expect(s.Code()).To(Equal(codes.FailedPrecondition)) + } + } + Expect(err).To(BeNil()) + Eventually(func() error { + status, err := alertClusterClient.GetClusterStatus(env.Context(), &emptypb.Empty{}) + if err != nil { return err - }, time.Second*5, time.Millisecond*200).Should(Succeed()) - }) + } + if status.State != alertops.InstallState_Installed { + return fmt.Errorf("alerting cluster install state is %s", status.State.String()) + } + return nil + }, time.Second*5, time.Millisecond*200).Should(Succeed()) + + Eventually(func() error { + getConf, err := alertClusterClient.GetClusterConfiguration(env.Context(), &emptypb.Empty{}) + if !proto.Equal(getConf, clusterConf) { + return fmt.Errorf("cluster config not equal : not applied") + } + return err + }, time.Second*5, time.Millisecond*200).Should(Succeed()) + }) - Specify("the alerting plugin components should be running and healthy", func() { - alertingReadinessProbe := fmt.Sprintf("https://%s/plugin_alerting/ready", env.GatewayConfig().Spec.HTTPListenAddress) - alertingHealthProbe := fmt.Sprintf("https://%s/plugin_alerting/healthy", env.GatewayConfig().Spec.HTTPListenAddress) + Specify("the alerting plugin components should be running and healthy", func() { + alertingReadinessProbe := fmt.Sprintf("https://%s/plugin_alerting/ready", env.GatewayConfig().Spec.HTTPListenAddress) + alertingHealthProbe := fmt.Sprintf("https://%s/plugin_alerting/healthy", env.GatewayConfig().Spec.HTTPListenAddress) - Eventually(func() error { - reqReady, err := http.NewRequestWithContext(env.Context(), http.MethodGet, alertingReadinessProbe, nil) - if err != nil { - return err - } - reqHealthy, err := http.NewRequestWithContext(env.Context(), http.MethodGet, alertingHealthProbe, nil) - if err != nil { - return err - } - respReady, err := httpProxyClient.Do(reqReady) - if err != nil { - return err - } - defer respReady.Body.Close() - if respReady.StatusCode != http.StatusOK { - return fmt.Errorf("alerting plugin not yet ready %d, %s", respReady.StatusCode, respReady.Status) - } - respHealthy, err := httpProxyClient.Do(reqHealthy) + Eventually(func() error { + reqReady, err := http.NewRequestWithContext(env.Context(), http.MethodGet, alertingReadinessProbe, nil) + if err != nil { + return err + } + reqHealthy, err := http.NewRequestWithContext(env.Context(), http.MethodGet, alertingHealthProbe, nil) + if err != nil { + return err + } + respReady, err := httpProxyClient.Do(reqReady) + if err != nil { + return err + } + defer respReady.Body.Close() + if respReady.StatusCode != http.StatusOK { + return fmt.Errorf("alerting plugin not yet ready %d, %s", respReady.StatusCode, respReady.Status) + } + respHealthy, err := httpProxyClient.Do(reqHealthy) + if err != nil { + return err + } + defer respHealthy.Body.Close() + if respHealthy.StatusCode != http.StatusOK { if err != nil { - return err - } - defer respHealthy.Body.Close() - if respHealthy.StatusCode != http.StatusOK { - if err != nil { - return fmt.Errorf("alerting plugin unhealthy %d, %s", respHealthy.StatusCode, respHealthy.Status) - } + return fmt.Errorf("alerting plugin unhealthy %d, %s", respHealthy.StatusCode, respHealthy.Status) } - return nil - }, time.Second*30, time.Second).Should(Succeed()) - }) - - It("should be able to create some endpoints", func() { - servers = alerting.CreateWebhookServer(env, numServers) - - for _, server := range servers { - ref, err := alertEndpointsClient.CreateAlertEndpoint(env.Context(), server.Endpoint()) - Expect(err).To(Succeed()) - server.EndpointId = ref.Id } + return nil + }, time.Second*30, time.Second).Should(Succeed()) + }) - By("verifying they are externally persisted") + It("should be able to create some endpoints", func() { + servers = alerting.CreateWebhookServer(env, numServers) - endpList, err := alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) + for _, server := range servers { + ref, err := alertEndpointsClient.CreateAlertEndpoint(env.Context(), server.Endpoint()) Expect(err).To(Succeed()) - Expect(endpList.Items).To(HaveLen(numServers)) + server.EndpointId = ref.Id + } - By("verifying they are reachable") + By("verifying they are externally persisted") - for _, endp := range endpList.GetItems() { - Expect(endp.GetId()).NotTo(BeNil()) - _, err := alertNotificationsClient.TestAlertEndpoint(env.Context(), endp.GetId()) - Expect(err).To(Succeed()) - } - maxSuccesses := 0 - Eventually(func() error { - success := 0 - errs := []error{} - for _, server := range servers { - if len(server.GetBuffer()) == 0 { - if success > maxSuccesses { - maxSuccesses = success - } - errs = append(errs, fmt.Errorf("server %v did not receive any alerts", server.Endpoint())) - } else { - success++ - } - } - if len(errs) > 0 { - return errors.Join(errs...) - } - return nil - }, time.Second*15, time.Millisecond*100).Should(Succeed(), fmt.Sprintf("only %d/%d servers received alerts", maxSuccesses, numServers)) + endpList, err := alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) + Expect(err).To(Succeed()) + Expect(endpList.Items).To(HaveLen(numServers)) + + By("verifying they are reachable") + for _, endp := range endpList.GetItems() { + Expect(endp.GetId()).NotTo(BeNil()) + _, err := alertNotificationsClient.TestAlertEndpoint(env.Context(), endp.GetId()) + Expect(err).To(Succeed()) + } + maxSuccesses := 0 + Eventually(func() error { + success := 0 + errs := []error{} for _, server := range servers { - server.ClearBuffer() + if len(server.GetBuffer()) == 0 { + if success > maxSuccesses { + maxSuccesses = success + } + errs = append(errs, fmt.Errorf("server %v did not receive any alerts", server.Endpoint())) + } else { + success++ + } } - }) + if len(errs) > 0 { + return errors.Join(errs...) + } + return nil + }, time.Second*15, time.Millisecond*100).Should(Succeed(), fmt.Sprintf("only %d/%d servers received alerts", maxSuccesses, numServers)) - It("should create some default conditions when bootstrapping agents", func() { - By("expecting to have no initial conditions") - Eventually(func() error { - condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) - if err != nil { - return err - } - if len(condList.Items) != 0 { - return fmt.Errorf("expected 0 conditions, got %d", len(condList.Items)) - } - return nil - }, time.Second*30, time.Millisecond*200).Should(Succeed()) + for _, server := range servers { + server.ClearBuffer() + } + }) - By(fmt.Sprintf("bootstrapping %d agents", numAgents)) - certsInfo, err := mgmtClient.CertsInfo(context.Background(), &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - fingerprint := certsInfo.Chain[len(certsInfo.Chain)-1].Fingerprint - Expect(fingerprint).NotTo(BeEmpty()) + It("should create some default conditions when bootstrapping agents", func() { + By("expecting to have no initial conditions") + Eventually(func() error { + condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + if err != nil { + return err + } + if len(condList.Items) != 0 { + return fmt.Errorf("expected 0 conditions, got %d", len(condList.Items)) + } + return nil + }, time.Second*30, time.Millisecond*200).Should(Succeed()) - token, err := mgmtClient.CreateBootstrapToken(context.Background(), &managementv1.CreateBootstrapTokenRequest{ - Ttl: durationpb.New(1 * time.Hour), + By(fmt.Sprintf("bootstrapping %d agents", numAgents)) + certsInfo, err := mgmtClient.CertsInfo(context.Background(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + fingerprint := certsInfo.Chain[len(certsInfo.Chain)-1].Fingerprint + Expect(fingerprint).NotTo(BeEmpty()) + + token, err := mgmtClient.CreateBootstrapToken(context.Background(), &managementv1.CreateBootstrapTokenRequest{ + Ttl: durationpb.New(1 * time.Hour), + }) + Expect(err).NotTo(HaveOccurred()) + agentIdFunc := func(i int) string { + return fmt.Sprintf("agent-%d-%s", i, uuid.New().String()) + } + agents = []*agentWithContext{} + for i := 0; i < numAgents; i++ { + ctxCa, ca := context.WithCancel(env.Context()) + id := agentIdFunc(i) + port, errC := env.StartAgent(id, token, []string{fingerprint}, test.WithContext(ctxCa)) + Eventually(errC).Should(Receive(BeNil())) + agents = append(agents, &agentWithContext{ + port: port, + CancelFunc: ca, + Context: ctxCa, + id: id, }) - Expect(err).NotTo(HaveOccurred()) - agentIdFunc := func(i int) string { - return fmt.Sprintf("agent-%d-%s", i, uuid.New().String()) - } - agents = []*agentWithContext{} - for i := 0; i < numAgents; i++ { - ctxCa, ca := context.WithCancel(env.Context()) - id := agentIdFunc(i) - port, errC := env.StartAgent(id, token, []string{fingerprint}, test.WithContext(ctxCa)) - Eventually(errC).Should(Receive(BeNil())) - agents = append(agents, &agentWithContext{ - port: port, - CancelFunc: ca, - Context: ctxCa, - id: id, - }) + } + By("verifying that there are default conditions") + Eventually(func() error { + condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + if err != nil { + return err + } + if len(condList.Items) != numAgents*2 { + return fmt.Errorf("expected %d conditions, got %d", numAgents*2, len(condList.Items)) } - By("verifying that there are default conditions") + return nil + }, time.Second*5, time.Millisecond*200).Should(Succeed()) + + By("verifying that only the default group exists") + groupList, err := alertConditionsClient.ListAlertConditionGroups(env.Context(), &emptypb.Empty{}) + Expect(err).To(Succeed()) + Expect(groupList.Items).To(HaveLen(1)) + Expect(groupList.Items[0].Id).To(Equal("")) + }) + + It("shoud list conditions by given filters", func() { + for _, agent := range agents { Eventually(func() error { - condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + filteredByCluster, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ + Clusters: []string{agent.id}, + }) if err != nil { return err } - if len(condList.Items) != numAgents*2 { - return fmt.Errorf("expected %d conditions, got %d", numAgents*2, len(condList.Items)) + if len(filteredByCluster.Items) != 2 { + return fmt.Errorf("expected 2 conditions, got %d", len(filteredByCluster.Items)) } return nil - }, time.Second*5, time.Millisecond*200).Should(Succeed()) + }, time.Second).Should(Succeed()) - By("verifying that only the default group exists") - groupList, err := alertConditionsClient.ListAlertConditionGroups(env.Context(), &emptypb.Empty{}) - Expect(err).To(Succeed()) - Expect(groupList.Items).To(HaveLen(1)) - Expect(groupList.Items[0].Id).To(Equal("")) - }) - - It("shoud list conditions by given filters", func() { - for _, agent := range agents { - Eventually(func() error { - filteredByCluster, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ - Clusters: []string{agent.id}, - }) - if err != nil { - return err - } - if len(filteredByCluster.Items) != 2 { - return fmt.Errorf("expected 2 conditions, got %d", len(filteredByCluster.Items)) - } - return nil - }, time.Second).Should(Succeed()) - - } + } - By("verifying all the agents conditions are critical") + By("verifying all the agents conditions are critical") - filterList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ - Severities: []alertingv1.OpniSeverity{ - alertingv1.OpniSeverity_Warning, - alertingv1.OpniSeverity_Error, - alertingv1.OpniSeverity_Info, - }, - }) - Expect(err).To(Succeed()) - Expect(filterList.Items).To(HaveLen(0)) + filterList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ + Severities: []alertingv1.OpniSeverity{ + alertingv1.OpniSeverity_Warning, + alertingv1.OpniSeverity_Error, + alertingv1.OpniSeverity_Info, + }, + }) + Expect(err).To(Succeed()) + Expect(filterList.Items).To(HaveLen(0)) - By("verifying we have an equal number of disconnect and capability unhealthy") + By("verifying we have an equal number of disconnect and capability unhealthy") - disconnectList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ + disconnectList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ + AlertTypes: []alertingv1.AlertType{ + alertingv1.AlertType_System, + }, + }) + Expect(err).To(Succeed()) + disconnectStatusList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{ + ItemFilter: &alertingv1.ListAlertConditionRequest{ AlertTypes: []alertingv1.AlertType{ alertingv1.AlertType_System, }, - }) - Expect(err).To(Succeed()) - disconnectStatusList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{ - ItemFilter: &alertingv1.ListAlertConditionRequest{ - AlertTypes: []alertingv1.AlertType{ - alertingv1.AlertType_System, - }, - }, - }) - Expect(err).To(Succeed()) + }, + }) + Expect(err).To(Succeed()) + + capabilityList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ + AlertTypes: []alertingv1.AlertType{ + alertingv1.AlertType_DownstreamCapability, + }, + }) + Expect(err).To(Succeed()) - capabilityList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ + capabilityStatusList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{ + ItemFilter: &alertingv1.ListAlertConditionRequest{ AlertTypes: []alertingv1.AlertType{ alertingv1.AlertType_DownstreamCapability, }, - }) - Expect(err).To(Succeed()) - - capabilityStatusList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{ - ItemFilter: &alertingv1.ListAlertConditionRequest{ - AlertTypes: []alertingv1.AlertType{ - alertingv1.AlertType_DownstreamCapability, - }, - }, - }) - - Expect(err).To(Succeed()) - Expect(capabilityList.Items).To(HaveLen(len(disconnectList.Items))) - Expect(capabilityStatusList.GetAlertConditions()).To(HaveLen(len(disconnectStatusList.GetAlertConditions()))) + }, }) - It("should be able to attach endpoints to conditions", func() { - By("attaching a sample of random endpoints to default agent conditions") - condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + Expect(err).To(Succeed()) + Expect(capabilityList.Items).To(HaveLen(len(disconnectList.Items))) + Expect(capabilityStatusList.GetAlertConditions()).To(HaveLen(len(disconnectStatusList.GetAlertConditions()))) + }) + + It("should be able to attach endpoints to conditions", func() { + By("attaching a sample of random endpoints to default agent conditions") + condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + Expect(err).To(Succeed()) + endpList, err := alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) + for _, cond := range condList.Items { Expect(err).To(Succeed()) - endpList, err := alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) - for _, cond := range condList.Items { - Expect(err).To(Succeed()) - endps := lo.Map( - lo.Samples(endpList.Items, 1+rand.Intn(len(endpList.Items)-1)), - func(a *alertingv1.AlertEndpointWithId, _ int) *alertingv1.AttachedEndpoint { - return &alertingv1.AttachedEndpoint{ - EndpointId: a.GetId().Id, - } - }) + endps := lo.Map( + lo.Samples(endpList.Items, 1+rand.Intn(len(endpList.Items)-1)), + func(a *alertingv1.AlertEndpointWithId, _ int) *alertingv1.AttachedEndpoint { + return &alertingv1.AttachedEndpoint{ + EndpointId: a.GetId().Id, + } + }) - if cond.GetAlertCondition().GetAlertType().GetSystem() != nil { - for _, endp := range endps { - if _, ok := expectedRouting[endp.EndpointId]; !ok { - expectedRouting[endp.EndpointId] = &alertingv1.ConditionReferenceList{ - Items: []*alertingv1.ConditionReference{}, - } + if cond.GetAlertCondition().GetAlertType().GetSystem() != nil { + for _, endp := range endps { + if _, ok := expectedRouting[endp.EndpointId]; !ok { + expectedRouting[endp.EndpointId] = &alertingv1.ConditionReferenceList{ + Items: []*alertingv1.ConditionReference{}, } - expectedRouting[endp.EndpointId].Items = append(expectedRouting[endp.EndpointId].Items, cond.GetId()) } - cond.AlertCondition.AttachedEndpoints = &alertingv1.AttachedEndpoints{ - Items: endps, - InitialDelay: durationpb.New(time.Second * 1), - ThrottlingDuration: durationpb.New(time.Second * 1), - Details: &alertingv1.EndpointImplementation{ - Title: "disconnected agent", - Body: "agent %s is disconnected", - }, - } - _, err = alertConditionsClient.UpdateAlertCondition(env.Context(), &alertingv1.UpdateAlertConditionRequest{ - Id: cond.GetId(), - UpdateAlert: cond.AlertCondition, - }) - Expect(err).To(Succeed()) + expectedRouting[endp.EndpointId].Items = append(expectedRouting[endp.EndpointId].Items, cond.GetId()) + } + cond.AlertCondition.AttachedEndpoints = &alertingv1.AttachedEndpoints{ + Items: endps, + InitialDelay: durationpb.New(time.Second * 1), + ThrottlingDuration: durationpb.New(time.Second * 1), + Details: &alertingv1.EndpointImplementation{ + Title: "disconnected agent", + Body: "agent %s is disconnected", + }, } - } - for _, refs := range expectedRouting { - slices.SortFunc(refs.Items, func(a, b *alertingv1.ConditionReference) bool { - if a.GroupId != b.GroupId { - return a.GroupId < b.GroupId - } - return a.Id < b.Id + _, err = alertConditionsClient.UpdateAlertCondition(env.Context(), &alertingv1.UpdateAlertConditionRequest{ + Id: cond.GetId(), + UpdateAlert: cond.AlertCondition, }) - } - - By("creating some default webhook servers as endpoints") - notificationServers = alerting.CreateWebhookServer(env, numNotificationServers) - for _, server := range notificationServers { - ref, err := alertEndpointsClient.CreateAlertEndpoint(env.Context(), server.Endpoint()) Expect(err).To(Succeed()) - server.EndpointId = ref.Id } - endpList, err = alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) - Expect(err).To(Succeed()) - Expect(endpList.Items).To(HaveLen(numNotificationServers + numServers)) + } + for _, refs := range expectedRouting { + slices.SortFunc(refs.Items, func(a, b *alertingv1.ConditionReference) bool { + if a.GroupId != b.GroupId { + return a.GroupId < b.GroupId + } + return a.Id < b.Id + }) + } - By("setting the default servers as default endpoints") - for _, server := range notificationServers { - _, err = alertEndpointsClient.ToggleNotifications(env.Context(), &alertingv1.ToggleRequest{ - Id: &corev1.Reference{Id: server.EndpointId}, - }) - Expect(err).To(Succeed()) - } + By("creating some default webhook servers as endpoints") + notificationServers = alerting.CreateWebhookServer(env, numNotificationServers) + for _, server := range notificationServers { + ref, err := alertEndpointsClient.CreateAlertEndpoint(env.Context(), server.Endpoint()) + Expect(err).To(Succeed()) + server.EndpointId = ref.Id + } + endpList, err = alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) + Expect(err).To(Succeed()) + Expect(endpList.Items).To(HaveLen(numNotificationServers + numServers)) + + By("setting the default servers as default endpoints") + for _, server := range notificationServers { + _, err = alertEndpointsClient.ToggleNotifications(env.Context(), &alertingv1.ToggleRequest{ + Id: &corev1.Reference{Id: server.EndpointId}, + }) + Expect(err).To(Succeed()) + } - By("expecting the conditions to eventually move to the 'OK' state") - Eventually(func() error { - for _, cond := range condList.Items { - status, err := alertConditionsClient.AlertConditionStatus(env.Context(), cond.Id) - if err != nil { - return err - } - if status.State != alertingv1.AlertConditionState_Ok { - return fmt.Errorf("condition %s \"%s\" is not OK, instead in state %s, %s", cond.AlertCondition.Id, cond.AlertCondition.Name, status.State.String(), status.Reason) - } - } - return nil - }, time.Second*30, time.Second).Should(Succeed()) - By("verifying the routing relationships are correctly loaded") - Eventually(func() int { - relationships, err := alertNotificationsClient.ListRoutingRelationships(env.Context(), &emptypb.Empty{}) + By("expecting the conditions to eventually move to the 'OK' state") + Eventually(func() error { + for _, cond := range condList.Items { + status, err := alertConditionsClient.AlertConditionStatus(env.Context(), cond.Id) if err != nil { - return -1 + return err + } + if status.State != alertingv1.AlertConditionState_Ok { + return fmt.Errorf("condition %s \"%s\" is not OK, instead in state %s, %s", cond.AlertCondition.Id, cond.AlertCondition.Name, status.State.String(), status.Reason) } - return len(relationships.RoutingRelationships) - }).Should(Equal(len(expectedRouting))) + } + return nil + }, time.Second*30, time.Second).Should(Succeed()) + By("verifying the routing relationships are correctly loaded") + Eventually(func() int { relationships, err := alertNotificationsClient.ListRoutingRelationships(env.Context(), &emptypb.Empty{}) - Expect(err).To(Succeed()) + if err != nil { + return -1 + } + return len(relationships.RoutingRelationships) + }).Should(Equal(len(expectedRouting))) + relationships, err := alertNotificationsClient.ListRoutingRelationships(env.Context(), &emptypb.Empty{}) + Expect(err).To(Succeed()) - for endpId, rel := range relationships.RoutingRelationships { - slices.SortFunc(rel.Items, func(a, b *alertingv1.ConditionReference) bool { - if a.GroupId != b.GroupId { - return a.GroupId < b.GroupId - } - return a.Id < b.Id - }) - if _, ok := expectedRouting[endpId]; !ok { - Fail(fmt.Sprintf("Expected a routing relation to exist for endpoint id %s", endpId)) - } - Expect(len(rel.Items)).To(Equal(len(expectedRouting[endpId].Items))) - for i := range rel.Items { - Expect(rel.Items[i].Id).To(Equal(expectedRouting[endpId].Items[i].Id)) - Expect(rel.Items[i].GroupId).To(Equal(expectedRouting[endpId].Items[i].GroupId)) + for endpId, rel := range relationships.RoutingRelationships { + slices.SortFunc(rel.Items, func(a, b *alertingv1.ConditionReference) bool { + if a.GroupId != b.GroupId { + return a.GroupId < b.GroupId } + return a.Id < b.Id + }) + if _, ok := expectedRouting[endpId]; !ok { + Fail(fmt.Sprintf("Expected a routing relation to exist for endpoint id %s", endpId)) } - }) + Expect(len(rel.Items)).To(Equal(len(expectedRouting[endpId].Items))) + for i := range rel.Items { + Expect(rel.Items[i].Id).To(Equal(expectedRouting[endpId].Items[i].Id)) + Expect(rel.Items[i].GroupId).To(Equal(expectedRouting[endpId].Items[i].GroupId)) + } + } + }) - Specify("agent disconnect alarms should fire when agents are disconnected ", func() { - // Disconnect a random 3 agents, and verify the servers have the messages - By("disconnecting a random 3 agents") - disconnectedIds := []string{} - toDisconnect := lo.Samples(agents, 3) - for _, disc := range toDisconnect { - disc.CancelFunc() - disconnectedIds = append(disconnectedIds, disc.id) + Specify("agent disconnect alarms should fire when agents are disconnected ", func() { + // Disconnect a random 3 agents, and verify the servers have the messages + By("disconnecting a random 3 agents") + disconnectedIds := []string{} + toDisconnect := lo.Samples(agents, 3) + for _, disc := range toDisconnect { + disc.CancelFunc() + disconnectedIds = append(disconnectedIds, disc.id) + } + + condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + Expect(err).To(Succeed()) + notInvolvedDisconnects := map[string]struct{}{} + for _, cond := range condList.Items { + if cond.GetAlertCondition().GetAlertType().GetSystem() != nil { + if slices.Contains(disconnectedIds, cond.GetAlertCondition().GetAlertType().GetSystem().ClusterId.Id) { + endps := lo.Map(cond.AlertCondition.AttachedEndpoints.Items, + func(a *alertingv1.AttachedEndpoint, _ int) string { + return a.EndpointId + }) + involvedDisconnects[cond.GetAlertCondition().Id] = endps + } else { + notInvolvedDisconnects[cond.GetAlertCondition().Id] = struct{}{} + } } + } + webhooks := lo.Uniq(lo.Flatten(lo.Values(involvedDisconnects))) + Expect(len(webhooks)).To(BeNumerically(">", 0)) - condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) - Expect(err).To(Succeed()) - notInvolvedDisconnects := map[string]struct{}{} - for _, cond := range condList.Items { - if cond.GetAlertCondition().GetAlertType().GetSystem() != nil { - if slices.Contains(disconnectedIds, cond.GetAlertCondition().GetAlertType().GetSystem().ClusterId.Id) { - endps := lo.Map(cond.AlertCondition.AttachedEndpoints.Items, - func(a *alertingv1.AttachedEndpoint, _ int) string { - return a.EndpointId - }) - involvedDisconnects[cond.GetAlertCondition().Id] = endps - } else { - notInvolvedDisconnects[cond.GetAlertCondition().Id] = struct{}{} + By("verifying the agents are actually disconnected") + Eventually(func() error { + clusters, err := mgmtClient.ListClusters(env.Context(), &managementv1.ListClustersRequest{}) + if err != nil { + return err + } + for _, cl := range clusters.Items { + if slices.Contains(disconnectedIds, cl.GetId()) { + healthStatus, err := mgmtClient.GetClusterHealthStatus(env.Context(), cl.Reference()) + if err != nil { + return err + } + if !healthStatus.Status.Connected == false { + return fmt.Errorf("expected disconnected health status for cluster %s: %s", cl.GetId(), healthStatus.Status.String()) } } } - webhooks := lo.Uniq(lo.Flatten(lo.Values(involvedDisconnects))) - Expect(len(webhooks)).To(BeNumerically(">", 0)) - - By("verifying the agents are actually disconnected") - Eventually(func() error { - clusters, err := mgmtClient.ListClusters(env.Context(), &managementv1.ListClustersRequest{}) + return nil + }, 5*time.Second, 200*time.Millisecond).Should(Succeed()) + + By("verifying the physical servers have received the disconnect messages") + Eventually(func() error { + servers := servers + conditionIds := lo.Keys(involvedDisconnects) + for _, id := range conditionIds { + status, err := alertConditionsClient.AlertConditionStatus(env.Context(), &alertingv1.ConditionReference{Id: id}) if err != nil { return err } - for _, cl := range clusters.Items { - if slices.Contains(disconnectedIds, cl.GetId()) { - healthStatus, err := mgmtClient.GetClusterHealthStatus(env.Context(), cl.Reference()) - if err != nil { - return err - } - if !healthStatus.Status.Connected == false { - return fmt.Errorf("expected disconnected health status for cluster %s: %s", cl.GetId(), healthStatus.Status.String()) - } - } + if status.GetState() != alertingv1.AlertConditionState_Firing { + return fmt.Errorf("expected alerting condition %s to be firing, got %s", id, status.GetState().String()) } - return nil - }, 5*time.Second, 200*time.Millisecond).Should(Succeed()) + } - By("verifying the physical servers have received the disconnect messages") - Eventually(func() error { - servers := servers - conditionIds := lo.Keys(involvedDisconnects) - for _, id := range conditionIds { - status, err := alertConditionsClient.AlertConditionStatus(env.Context(), &alertingv1.ConditionReference{Id: id}) - if err != nil { - return err - } - if status.GetState() != alertingv1.AlertConditionState_Firing { - return fmt.Errorf("expected alerting condition %s to be firing, got %s", id, status.GetState().String()) - } + for id := range notInvolvedDisconnects { + status, err := alertConditionsClient.AlertConditionStatus(env.Context(), &alertingv1.ConditionReference{Id: id}) + if err != nil { + return err } - - for id := range notInvolvedDisconnects { - status, err := alertConditionsClient.AlertConditionStatus(env.Context(), &alertingv1.ConditionReference{Id: id}) - if err != nil { - return err - } - if status.GetState() != alertingv1.AlertConditionState_Ok { - return fmt.Errorf("expected unaffected alerting condition %s to be ok, got %s", id, status.GetState().String()) - } + if status.GetState() != alertingv1.AlertConditionState_Ok { + return fmt.Errorf("expected unaffected alerting condition %s to be ok, got %s", id, status.GetState().String()) } + } - for _, server := range servers { - if slices.Contains(webhooks, server.EndpointId) { - // hard to map these excatly without recreating the internal routing logic from the routers - // since we have dedicated routing integration tests, we can just check that the buffer is not empty - if len(server.GetBuffer()) == 0 { - return fmt.Errorf("expected webhook server %s to have messages, got %d", server.EndpointId, len(server.GetBuffer())) - } + for _, server := range servers { + if slices.Contains(webhooks, server.EndpointId) { + // hard to map these excatly without recreating the internal routing logic from the routers + // since we have dedicated routing integration tests, we can just check that the buffer is not empty + if len(server.GetBuffer()) == 0 { + return fmt.Errorf("expected webhook server %s to have messages, got %d", server.EndpointId, len(server.GetBuffer())) } } - return nil - }, time.Second*60, time.Millisecond*500).Should(Succeed()) + } + return nil + }, time.Second*60, time.Millisecond*500).Should(Succeed()) - By("verifying the notification servers have not received any alarm disconnect messages") - Eventually(func() error { - for _, server := range notificationServers { - if len(server.GetBuffer()) != 0 { - return fmt.Errorf("expected webhook server %s to not have any notifications, got %d", server.EndpointId, len(server.GetBuffer())) - } + By("verifying the notification servers have not received any alarm disconnect messages") + Eventually(func() error { + for _, server := range notificationServers { + if len(server.GetBuffer()) != 0 { + return fmt.Errorf("expected webhook server %s to not have any notifications, got %d", server.EndpointId, len(server.GetBuffer())) } - return nil - }, time.Second*5, time.Second*1).Should(Succeed()) - }) + } + return nil + }, time.Second*5, time.Second*1).Should(Succeed()) + }) - It("should be able to batch list status and filter by status", func() { - condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) - Expect(err).To(Succeed()) + It("should be able to batch list status and filter by status", func() { + condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + Expect(err).To(Succeed()) - statusCondList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{}) - Expect(err).To(Succeed()) - Expect(statusCondList.AlertConditions).To(HaveLen(len(condList.Items))) - for condId, cond := range statusCondList.AlertConditions { - if slices.Contains(lo.Keys(involvedDisconnects), condId) { - Expect(cond.Status.State).To(Equal(alertingv1.AlertConditionState_Firing)) - } else { - Expect(cond.Status.State).To(Equal(alertingv1.AlertConditionState_Ok)) - } - } - firingOnlyStatusList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{ - States: []alertingv1.AlertConditionState{ - alertingv1.AlertConditionState_Firing, - }, - }) - Expect(err).To(Succeed()) - Expect(firingOnlyStatusList.AlertConditions).To(HaveLen(len(involvedDisconnects))) - for _, cond := range firingOnlyStatusList.AlertConditions { + statusCondList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{}) + Expect(err).To(Succeed()) + Expect(statusCondList.AlertConditions).To(HaveLen(len(condList.Items))) + for condId, cond := range statusCondList.AlertConditions { + if slices.Contains(lo.Keys(involvedDisconnects), condId) { Expect(cond.Status.State).To(Equal(alertingv1.AlertConditionState_Firing)) + } else { + Expect(cond.Status.State).To(Equal(alertingv1.AlertConditionState_Ok)) } + } + firingOnlyStatusList, err := alertConditionsClient.ListAlertConditionsWithStatus(env.Context(), &alertingv1.ListStatusRequest{ + States: []alertingv1.AlertConditionState{ + alertingv1.AlertConditionState_Firing, + }, }) + Expect(err).To(Succeed()) + Expect(firingOnlyStatusList.AlertConditions).To(HaveLen(len(involvedDisconnects))) + for _, cond := range firingOnlyStatusList.AlertConditions { + Expect(cond.Status.State).To(Equal(alertingv1.AlertConditionState_Firing)) + } + }) - It("should be able to push notifications to our notification endpoints", func() { - Expect(len(notificationServers)).To(BeNumerically(">", 0)) - By("forwarding the message to AlertManager") - _, err := alertNotificationsClient.PushNotification(env.Context(), &alertingv1.Notification{ - Title: "hello", - Body: "world", - // set to critical in order to expedite the notification during testing - Properties: map[string]string{ - message.NotificationPropertySeverity: alertingv1.OpniSeverity_Critical.String(), - }, - }) - Expect(err).To(Succeed()) - - By("verifying the endpoints have received the notification messages") - Eventually(func() error { - for _, server := range notificationServers { - if len(server.GetBuffer()) == 0 { - return fmt.Errorf("expected webhook server %s to have messages, got %d", server.EndpointId, len(server.GetBuffer())) - } - } - return nil - }, time.Second*60, time.Second).Should(Succeed()) + It("should be able to push notifications to our notification endpoints", func() { + Expect(len(notificationServers)).To(BeNumerically(">", 0)) + By("forwarding the message to AlertManager") + _, err := alertNotificationsClient.PushNotification(env.Context(), &alertingv1.Notification{ + Title: "hello", + Body: "world", + // set to critical in order to expedite the notification during testing + Properties: map[string]string{ + message.NotificationPropertySeverity: alertingv1.OpniSeverity_Critical.String(), + }, }) + Expect(err).To(Succeed()) - It("should be able to list opni messages", func() { - Eventually(func() error { - list, err := alertNotificationsClient.ListNotifications(env.Context(), &alertingv1.ListNotificationRequest{}) - if err != nil { - return err - } - if len(list.Items) == 0 { - return fmt.Errorf("expected to find at least one notification, got 0") + By("verifying the endpoints have received the notification messages") + Eventually(func() error { + for _, server := range notificationServers { + if len(server.GetBuffer()) == 0 { + return fmt.Errorf("expected webhook server %s to have messages, got %d", server.EndpointId, len(server.GetBuffer())) } - return nil - }, time.Second*60, time.Second).Should(Succeed()) + } + return nil + }, time.Second*60, time.Second).Should(Succeed()) + }) - By("verifying we enforce limits") - list, err := alertNotificationsClient.ListNotifications(env.Context(), &alertingv1.ListNotificationRequest{ - Limit: lo.ToPtr(int32(1)), - }) - Expect(err).To(Succeed()) - Expect(len(list.Items)).To(Equal(1)) + It("should be able to list opni messages", func() { + Eventually(func() error { + list, err := alertNotificationsClient.ListNotifications(env.Context(), &alertingv1.ListNotificationRequest{}) + if err != nil { + return err + } + if len(list.Items) == 0 { + return fmt.Errorf("expected to find at least one notification, got 0") + } + return nil + }, time.Second*60, time.Second).Should(Succeed()) + + By("verifying we enforce limits") + list, err := alertNotificationsClient.ListNotifications(env.Context(), &alertingv1.ListNotificationRequest{ + Limit: lo.ToPtr(int32(1)), }) + Expect(err).To(Succeed()) + Expect(len(list.Items)).To(Equal(1)) + }) - It("should return warnings when trying to edit/delete alert endpoints that are involved in conditions", func() { - webhooks := lo.Uniq(lo.Flatten(lo.Values(involvedDisconnects))) - Expect(len(webhooks)).To(BeNumerically(">", 0)) + It("should return warnings when trying to edit/delete alert endpoints that are involved in conditions", func() { + webhooks := lo.Uniq(lo.Flatten(lo.Values(involvedDisconnects))) + Expect(len(webhooks)).To(BeNumerically(">", 0)) - for _, webhook := range webhooks { - involvedConditions, err := alertEndpointsClient.UpdateAlertEndpoint(env.Context(), &alertingv1.UpdateAlertEndpointRequest{ - Id: &corev1.Reference{ - Id: webhook, - }, - UpdateAlert: &alertingv1.AlertEndpoint{ - Name: "update", - Description: "update", - Endpoint: &alertingv1.AlertEndpoint_Webhook{ - Webhook: &alertingv1.WebhookEndpoint{ - Url: "http://example.com", - }, + for _, webhook := range webhooks { + involvedConditions, err := alertEndpointsClient.UpdateAlertEndpoint(env.Context(), &alertingv1.UpdateAlertEndpointRequest{ + Id: &corev1.Reference{ + Id: webhook, + }, + UpdateAlert: &alertingv1.AlertEndpoint{ + Name: "update", + Description: "update", + Endpoint: &alertingv1.AlertEndpoint_Webhook{ + Webhook: &alertingv1.WebhookEndpoint{ + Url: "http://example.com", }, - Id: "id", }, - ForceUpdate: false, - }) - Expect(err).NotTo(HaveOccurred()) + Id: "id", + }, + ForceUpdate: false, + }) + Expect(err).NotTo(HaveOccurred()) - Expect(involvedConditions.Items).NotTo(HaveLen(0)) - involvedConditions, err = alertEndpointsClient.DeleteAlertEndpoint(env.Context(), &alertingv1.DeleteAlertEndpointRequest{ - Id: &corev1.Reference{ - Id: webhook, - }, - ForceDelete: false, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(involvedConditions.Items).NotTo(HaveLen(0)) - } - }) + Expect(involvedConditions.Items).NotTo(HaveLen(0)) + involvedConditions, err = alertEndpointsClient.DeleteAlertEndpoint(env.Context(), &alertingv1.DeleteAlertEndpointRequest{ + Id: &corev1.Reference{ + Id: webhook, + }, + ForceDelete: false, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(involvedConditions.Items).NotTo(HaveLen(0)) + } + }) - It("should have a functional timeline", func() { - condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) - Expect(err).To(Succeed()) + It("should have a functional timeline", func() { + condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + Expect(err).To(Succeed()) - // By("verifying the timeline shows only the firing conditions ") - Eventually(func() error { - timeline, err := alertConditionsClient.Timeline(env.Context(), &alertingv1.TimelineRequest{ - LookbackWindow: durationpb.New(time.Minute * 5), - }) - if err != nil { - return err - } + // By("verifying the timeline shows only the firing conditions ") + Eventually(func() error { + timeline, err := alertConditionsClient.Timeline(env.Context(), &alertingv1.TimelineRequest{ + LookbackWindow: durationpb.New(time.Minute * 5), + }) + if err != nil { + return err + } - By("verifying the timeline matches the conditions") - if len(timeline.Items) != len(condList.Items) { - return fmt.Errorf("expected timeline to have %d items, got %d", len(condList.Items), len(timeline.Items)) - } + By("verifying the timeline matches the conditions") + if len(timeline.Items) != len(condList.Items) { + return fmt.Errorf("expected timeline to have %d items, got %d", len(condList.Items), len(timeline.Items)) + } - for id, item := range timeline.GetItems() { - if slices.Contains(lo.Keys(involvedDisconnects), id) { - if len(item.Windows) == 0 { - return fmt.Errorf("firing condition should show up on timeline, but does not") - } - if len(item.Windows) != 1 { - return fmt.Errorf("condition evaluation is flaky, should only have one window, but has %d", len(item.Windows)) - } - messages, err := alertNotificationsClient.ListAlarmMessages(env.Context(), &alertingv1.ListAlarmMessageRequest{ - ConditionId: &alertingv1.ConditionReference{ - Id: id, - }, - Fingerprints: item.Windows[0].Fingerprints, - Start: item.Windows[0].Start, - End: timestamppb.Now(), - }) - if err != nil { - return err - } - if !(len(messages.Items) > 0) { - return fmt.Errorf("expected firing condition to have cached messages") - } - Expect(len(messages.Items)).To(BeNumerically(">", 0)) + for id, item := range timeline.GetItems() { + if slices.Contains(lo.Keys(involvedDisconnects), id) { + if len(item.Windows) == 0 { + return fmt.Errorf("firing condition should show up on timeline, but does not") + } + if len(item.Windows) != 1 { + return fmt.Errorf("condition evaluation is flaky, should only have one window, but has %d", len(item.Windows)) + } + messages, err := alertNotificationsClient.ListAlarmMessages(env.Context(), &alertingv1.ListAlarmMessageRequest{ + ConditionId: &alertingv1.ConditionReference{ + Id: id, + }, + Fingerprints: item.Windows[0].Fingerprints, + Start: item.Windows[0].Start, + End: timestamppb.Now(), + }) + if err != nil { + return err + } + if !(len(messages.Items) > 0) { + return fmt.Errorf("expected firing condition to have cached messages") + } + Expect(len(messages.Items)).To(BeNumerically(">", 0)) - } else { - if len(item.Windows) != 0 { - return fmt.Errorf("conditions that have not fired should not show up on timeline, but do") - } + } else { + if len(item.Windows) != 0 { + return fmt.Errorf("conditions that have not fired should not show up on timeline, but do") } } - return nil - }, time.Second*15, time.Second).Should(Succeed()) - }) + } + return nil + }, time.Second*15, time.Second).Should(Succeed()) + }) - Specify("the alertmanager proxy served by the Gateway HTTP port should be able to list the alarms", func() { - alertingProxyGET := fmt.Sprintf("https://%s/plugin_alerting/alertmanager/api/v2/alerts/groups", env.GatewayConfig().Spec.HTTPListenAddress) - req, err := http.NewRequestWithContext(env.Context(), http.MethodGet, alertingProxyGET, nil) - Expect(err).To(Succeed()) + Specify("the alertmanager proxy served by the Gateway HTTP port should be able to list the alarms", func() { + alertingProxyGET := fmt.Sprintf("https://%s/plugin_alerting/alertmanager/api/v2/alerts/groups", env.GatewayConfig().Spec.HTTPListenAddress) + req, err := http.NewRequestWithContext(env.Context(), http.MethodGet, alertingProxyGET, nil) + Expect(err).To(Succeed()) - Eventually(func() error { - resp, err := httpProxyClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("expected proxy to return status OK, instead got : %d, %s", resp.StatusCode, resp.Status) - } - res := alertmanagerv2.AlertGroups{} - if err := json.NewDecoder(resp.Body).Decode(&res); err != nil { - return err - } - if len(res) == 0 { - return fmt.Errorf("expected to get non-empty alertgroup from alertmanager") - } - return nil - }).Should(Succeed()) - }) + Eventually(func() error { + resp, err := httpProxyClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("expected proxy to return status OK, instead got : %d, %s", resp.StatusCode, resp.Status) + } + res := alertmanagerv2.AlertGroups{} + if err := json.NewDecoder(resp.Body).Decode(&res); err != nil { + return err + } + if len(res) == 0 { + return fmt.Errorf("expected to get non-empty alertgroup from alertmanager") + } + return nil + }).Should(Succeed()) + }) - It("should sync friendly cluster names to conditions", func() { - friendlyName := "friendly-name" - agentId := agents[0].id - cl, err := mgmtClient.GetCluster(env.Context(), &corev1.Reference{Id: agentId}) - Expect(err).To(Succeed()) + It("should sync friendly cluster names to conditions", func() { + friendlyName := "friendly-name" + agentId := agents[0].id + cl, err := mgmtClient.GetCluster(env.Context(), &corev1.Reference{Id: agentId}) + Expect(err).To(Succeed()) + + _, err = mgmtClient.EditCluster(env.Context(), &managementv1.EditClusterRequest{ + Cluster: &corev1.Reference{Id: agentId}, + Labels: lo.Assign(cl.GetLabels(), map[string]string{ + corev1.NameLabel: friendlyName, + }), + }) + Expect(err).To(Succeed()) - _, err = mgmtClient.EditCluster(env.Context(), &managementv1.EditClusterRequest{ - Cluster: &corev1.Reference{Id: agentId}, - Labels: lo.Assign(cl.GetLabels(), map[string]string{ - corev1.NameLabel: friendlyName, - }), + Eventually(func() error { + conds, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ + Clusters: []string{agentId}, }) - Expect(err).To(Succeed()) + if err != nil { + return err + } - Eventually(func() error { - conds, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{ - Clusters: []string{agentId}, - }) - if err != nil { - return err + for _, cond := range conds.Items { + name, ok := cond.GetAlertCondition().Annotations[message.NotificationContentClusterName] + if !ok { + return fmt.Errorf("expected alert condition to contain annotations with cluster name") } - - for _, cond := range conds.Items { - name, ok := cond.GetAlertCondition().Annotations[message.NotificationContentClusterName] - if !ok { - return fmt.Errorf("expected alert condition to contain annotations with cluster name") - } - if name != friendlyName { - return fmt.Errorf("expected alert condition to contain annotations with cluster name %s, got %s", friendlyName, name) - } + if name != friendlyName { + return fmt.Errorf("expected alert condition to contain annotations with cluster name %s, got %s", friendlyName, name) } - return nil - }, time.Second*5, time.Millisecond*300).Should(Succeed()) + } + return nil + }, time.Second*5, time.Millisecond*300).Should(Succeed()) - }) + }) - It("should force update/delete alert endpoints involved in conditions", func() { - By("verifying we can edit Alert Endpoints in use by Alert Conditions") - endpList, err := alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) - Expect(err).NotTo(HaveOccurred()) - Expect(len(endpList.Items)).To(BeNumerically(">", 0)) - Expect(endpList.Items).To(HaveLen(numServers + numNotificationServers)) - for _, endp := range endpList.Items { - _, err := alertEndpointsClient.UpdateAlertEndpoint(env.Context(), &alertingv1.UpdateAlertEndpointRequest{ - Id: &corev1.Reference{ - Id: endp.Id.Id, - }, - UpdateAlert: &alertingv1.AlertEndpoint{ - Name: "update", - Description: "update", - Endpoint: &alertingv1.AlertEndpoint_Webhook{ - Webhook: &alertingv1.WebhookEndpoint{ - Url: "http://example.com", - }, + It("should force update/delete alert endpoints involved in conditions", func() { + By("verifying we can edit Alert Endpoints in use by Alert Conditions") + endpList, err := alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(len(endpList.Items)).To(BeNumerically(">", 0)) + Expect(endpList.Items).To(HaveLen(numServers + numNotificationServers)) + for _, endp := range endpList.Items { + _, err := alertEndpointsClient.UpdateAlertEndpoint(env.Context(), &alertingv1.UpdateAlertEndpointRequest{ + Id: &corev1.Reference{ + Id: endp.Id.Id, + }, + UpdateAlert: &alertingv1.AlertEndpoint{ + Name: "update", + Description: "update", + Endpoint: &alertingv1.AlertEndpoint_Webhook{ + Webhook: &alertingv1.WebhookEndpoint{ + Url: "http://example.com", }, - Id: endp.Id.Id, }, - ForceUpdate: true, - }) - Expect(err).NotTo(HaveOccurred()) - } - endpList, err = alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) - Expect(err).NotTo(HaveOccurred()) - Expect(len(endpList.Items)).To(BeNumerically(">", 0)) - Expect(endpList.Items).To(HaveLen(numServers + numNotificationServers)) - updatedList := lo.Filter(endpList.Items, func(item *alertingv1.AlertEndpointWithId, _ int) bool { - if item.Endpoint.GetWebhook() != nil { - return item.Endpoint.GetName() == "update" && item.Endpoint.GetDescription() == "update" - } - return false + Id: endp.Id.Id, + }, + ForceUpdate: true, }) - Expect(updatedList).To(HaveLen(len(endpList.Items))) - - By("verifying we can delete Alert Endpoint in use by Alert Conditions") - for _, endp := range endpList.Items { - _, err := alertEndpointsClient.DeleteAlertEndpoint(env.Context(), &alertingv1.DeleteAlertEndpointRequest{ - Id: &corev1.Reference{ - Id: endp.Id.Id, - }, - ForceDelete: true, - }) - Expect(err).NotTo(HaveOccurred()) - } - endpList, err = alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) Expect(err).NotTo(HaveOccurred()) - Expect(endpList.Items).To(HaveLen(0)) - - condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) - Expect(err).NotTo(HaveOccurred()) - Expect(condList.Items).NotTo(HaveLen(0)) - hasEndpoints := lo.Filter(condList.Items, func(item *alertingv1.AlertConditionWithId, _ int) bool { - if item.AlertCondition.AttachedEndpoints != nil { - return len(item.AlertCondition.AttachedEndpoints.Items) != 0 - } - return false - }) - Expect(hasEndpoints).To(HaveLen(0)) + } + endpList, err = alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(len(endpList.Items)).To(BeNumerically(">", 0)) + Expect(endpList.Items).To(HaveLen(numServers + numNotificationServers)) + updatedList := lo.Filter(endpList.Items, func(item *alertingv1.AlertEndpointWithId, _ int) bool { + if item.Endpoint.GetWebhook() != nil { + return item.Endpoint.GetName() == "update" && item.Endpoint.GetDescription() == "update" + } + return false }) + Expect(updatedList).To(HaveLen(len(endpList.Items))) - It("should delete the downstream agents", func() { - client := env.NewManagementClient() - agents, err := client.ListClusters(env.Context(), &managementv1.ListClustersRequest{}) + By("verifying we can delete Alert Endpoint in use by Alert Conditions") + for _, endp := range endpList.Items { + _, err := alertEndpointsClient.DeleteAlertEndpoint(env.Context(), &alertingv1.DeleteAlertEndpointRequest{ + Id: &corev1.Reference{ + Id: endp.Id.Id, + }, + ForceDelete: true, + }) Expect(err).NotTo(HaveOccurred()) - for _, agent := range agents.Items { - _, err := client.DeleteCluster(env.Context(), agent.Reference()) - Expect(err).NotTo(HaveOccurred()) + } + endpList, err = alertEndpointsClient.ListAlertEndpoints(env.Context(), &alertingv1.ListAlertEndpointsRequest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(endpList.Items).To(HaveLen(0)) + + condList, err := alertConditionsClient.ListAlertConditions(env.Context(), &alertingv1.ListAlertConditionRequest{}) + Expect(err).NotTo(HaveOccurred()) + Expect(condList.Items).NotTo(HaveLen(0)) + hasEndpoints := lo.Filter(condList.Items, func(item *alertingv1.AlertConditionWithId, _ int) bool { + if item.AlertCondition.AttachedEndpoints != nil { + return len(item.AlertCondition.AttachedEndpoints.Items) != 0 } + return false }) + Expect(hasEndpoints).To(HaveLen(0)) + }) - It("should uninstall the alerting cluster", func() { - _, err := alertClusterClient.UninstallCluster(env.Context(), &alertops.UninstallRequest{ - DeleteData: true, - }) - Expect(err).To(BeNil()) + It("should delete the downstream agents", func() { + client := env.NewManagementClient() + agents, err := client.ListClusters(env.Context(), &managementv1.ListClustersRequest{}) + Expect(err).NotTo(HaveOccurred()) + for _, agent := range agents.Items { + _, err := client.DeleteCluster(env.Context(), agent.Reference()) + Expect(err).NotTo(HaveOccurred()) + } + }) + + It("should uninstall the alerting cluster", func() { + _, err := alertClusterClient.UninstallCluster(env.Context(), &alertops.UninstallRequest{ + DeleteData: true, }) - } - }) + Expect(err).To(BeNil()) + }) + } }) -} +}) diff --git a/test/plugins/alerting/output.log b/test/plugins/alerting/output.log new file mode 100644 index 0000000000..8f73deda13 --- /dev/null +++ b/test/plugins/alerting/output.log @@ -0,0 +1,20043 @@ +Running Suite: Alerting Suite - /home/ec2-user/opni/test/plugins/alerting +========================================================================= +Random Seed: 1694196603 + +Will run 11 of 54 specs +SSSSSSSSSSSSS•••• +------------------------------ +P [PENDING] +SLO Alerting when Managing SLOs with the mocked datasource when the SLOStore fails to create an SLO, the management server should not have it [integration] +/home/ec2-user/opni/test/plugins/alerting/slo_test.go:218 +------------------------------ +••• +------------------------------ +P [PENDING] +SLO Alerting when Managing SLOs with the mocked datasource should error when the fetching the status of non-existent SLO [integration] +/home/ec2-user/opni/test/plugins/alerting/slo_test.go:260 +------------------------------ +runtime: bad pointer in frame github.com/rancher/opni/test/plugins/alerting_test.glob..func6.2.10 at 0xc002d43ed0: 0x28 +fatal error: invalid pointer found on stack + +runtime stack: +runtime.throw({0x465c08a?, 0x755bb20?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/panic.go:1077 +0x5c fp=0x7f4223ffe838 sp=0x7f4223ffe808 pc=0x4748bc +runtime.adjustpointers(0x7f4223ffeab0?, 0x7f4223ffe8f8, 0x49cdc5?, {0x7f4223ffeab0?, 0x0?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/stack.go:627 +0x1ad fp=0x7f4223ffe898 sp=0x7f4223ffe838 pc=0x48fa0d +runtime.adjustframe(0x7f4223ffeab0, 0x7f4223ffe990) + /home/ec2-user/sdk/go1.21.0/src/runtime/stack.go:684 +0xdb fp=0x7f4223ffe928 sp=0x7f4223ffe898 pc=0x48fb3b +runtime.copystack(0xc002e956c0, 0x800000002?) + /home/ec2-user/sdk/go1.21.0/src/runtime/stack.go:935 +0x2c5 fp=0x7f4223ffec20 sp=0x7f4223ffe928 pc=0x4902e5 +runtime.newstack() + /home/ec2-user/sdk/go1.21.0/src/runtime/stack.go:1116 +0x47f fp=0x7f4223ffedd0 sp=0x7f4223ffec20 pc=0x49089f +runtime.morestack() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:593 +0x8f fp=0x7f4223ffedd8 sp=0x7f4223ffedd0 pc=0x4ab58f + +goroutine 1746 [copystack]: +runtime.checkptrBase(0xc00237e580) + /home/ec2-user/sdk/go1.21.0/src/runtime/checkptr.go:79 +0xd4 fp=0xc002d426c8 sp=0xc002d426c0 pc=0x441dd4 +runtime.checkptrArithmetic(0x0?, {0xc002d42720, 0x1, 0x4aead9?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/checkptr.go:58 +0x38 fp=0xc002d426f8 sp=0xc002d426c8 pc=0x441c38 +sync.indexLocal(0xc00237e400, 0x3) + /home/ec2-user/sdk/go1.21.0/src/sync/pool.go:280 +0x5c fp=0xc002d42738 sp=0xc002d426f8 pc=0x4bf2bc +sync.(*Pool).pin(0x770e520) + /home/ec2-user/sdk/go1.21.0/src/sync/pool.go:207 +0x4b fp=0xc002d42770 sp=0xc002d42738 pc=0x4beceb +sync.(*Pool).Get(0x770e520) + /home/ec2-user/sdk/go1.21.0/src/sync/pool.go:131 +0x35 fp=0xc002d427a8 sp=0xc002d42770 pc=0x4be975 +fmt.newPrinter() + /home/ec2-user/sdk/go1.21.0/src/fmt/print.go:152 +0x28 fp=0xc002d427d8 sp=0xc002d427a8 pc=0x591668 +fmt.Sprintf({0x4608983, 0x4}, {0xc002d42958, 0x1, 0x1}) + /home/ec2-user/sdk/go1.21.0/src/fmt/print.go:238 +0x3a fp=0xc002d42830 sp=0xc002d427d8 pc=0x591ffa +github.com/onsi/gomega/format.formatStruct({0x40ed880?, 0xc003195560?, 0x41a6d20?}, 0x5) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:478 +0x61d fp=0xc002d42a98 sp=0xc002d42830 pc=0xbaa91d +github.com/onsi/gomega/format.formatValue({0x40ed880, 0xc003195560, 0x1b9}, 0x5) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:384 +0xf2c fp=0xc002d42b80 sp=0xc002d42a98 pc=0xba8e2c +github.com/onsi/gomega/format.formatStruct({0x4315620?, 0xc003195560?, 0x41a6d20?}, 0x4) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:468 +0x22c fp=0xc002d42de8 sp=0xc002d42b80 pc=0xbaa52c +github.com/onsi/gomega/format.formatValue({0x4315620, 0xc003195560, 0x1b9}, 0x4) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:384 +0xf2c fp=0xc002d42ed0 sp=0xc002d42de8 pc=0xba8e2c +github.com/onsi/gomega/format.formatStruct({0x43aed20?, 0xc003195560?, 0x41a6d20?}, 0x3) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:468 +0x22c fp=0xc002d43138 sp=0xc002d42ed0 pc=0xbaa52c +github.com/onsi/gomega/format.formatValue({0x43aed20, 0xc003195560, 0x1b9}, 0x3) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:384 +0xf2c fp=0xc002d43220 sp=0xc002d43138 pc=0xba8e2c +github.com/onsi/gomega/format.formatValue({0x4330540, 0xc001f4eea8, 0x1b6}, 0x3) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:370 +0xcb4 fp=0xc002d43308 sp=0xc002d43220 pc=0xba8bb4 +github.com/onsi/gomega/format.formatStruct({0x4178aa0?, 0xc001f4eea8?, 0x41a6d20?}, 0x2) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:468 +0x22c fp=0xc002d43570 sp=0xc002d43308 pc=0xbaa52c +github.com/onsi/gomega/format.formatValue({0x4178aa0, 0xc001f4eea8, 0x1b9}, 0x2) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:384 +0xf2c fp=0xc002d43658 sp=0xc002d43570 pc=0xba8e2c +github.com/onsi/gomega/format.formatValue({0x42e29a0, 0xc00280eed0, 0x1b6}, 0x2) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:370 +0xcb4 fp=0xc002d43740 sp=0xc002d43658 pc=0xba8bb4 +github.com/onsi/gomega/format.formatStruct({0x4178b20?, 0xc00280eed0?, 0x41a6d20?}, 0x1) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:468 +0x22c fp=0xc002d439a8 sp=0xc002d43740 pc=0xbaa52c +github.com/onsi/gomega/format.formatValue({0x4178b20, 0xc00280eed0, 0x199}, 0x1) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:384 +0xf2c fp=0xc002d43a90 sp=0xc002d439a8 pc=0xba8e2c +github.com/onsi/gomega/format.formatValue({0x4178ba0, 0xc00280eed0, 0x16}, 0x1) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:370 +0xcb4 fp=0xc002d43b78 sp=0xc002d43a90 pc=0xba8bb4 +github.com/onsi/gomega/format.Object({0x4178ba0, 0xc00280eed0}, 0xc00280eed0?) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/format/format.go:265 +0x285 fp=0xc002d43cc0 sp=0xc002d43b78 pc=0xba72a5 +github.com/onsi/gomega/matchers.(*HaveOccurredMatcher).NegatedFailureMessage(0x7f42215d6dd0?, {0x4178ba0, 0xc00280eed0}) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/matchers/have_occurred_matcher.go:34 +0x3a fp=0xc002d43d28 sp=0xc002d43cc0 pc=0xc7f05a +github.com/onsi/gomega/internal.(*Assertion).match(0xc003079280, {0x5b29f98, 0x779c980}, 0x0, {0x0, 0x0, 0x0}) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/internal/assertion.go:103 +0x1d6 fp=0xc002d43df8 sp=0xc002d43d28 pc=0xbabc36 +github.com/onsi/gomega/internal.(*Assertion).NotTo(0xc003079280, {0x5b29f98, 0x779c980}, {0x0, 0x0, 0x0}) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/internal/assertion.go:74 +0x11e fp=0xc002d43e58 sp=0xc002d43df8 pc=0xbab85e +github.com/rancher/opni/test/plugins/alerting_test.glob..func6.2.10() + /home/ec2-user/opni/test/plugins/alerting/slo_test.go:273 +0x1bb fp=0xc002d43f00 sp=0xc002d43e58 pc=0x3dfd27b +github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func3({0x0, 0xc0030fecd0}) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/node.go:463 +0x2f fp=0xc002d43f20 sp=0xc002d43f00 pc=0xb749cf +github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3() + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/suite.go:865 +0x106 fp=0xc002d43fe0 sp=0xc002d43f20 pc=0xb956c6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d43fe8 sp=0xc002d43fe0 pc=0x4ad421 +created by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 85 + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/suite.go:852 +0x1345 + +goroutine 1 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x60?, 0x78?, 0x18?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00211f6a8 sp=0xc00211f688 pc=0x4777ae +runtime.chanrecv(0xc000cb4150, 0xc000bd178f, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00211f720 sp=0xc00211f6a8 pc=0x441325 +runtime.chanrecv1(0x4552dc0?, 0x3f99540?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00211f748 sp=0xc00211f720 pc=0x440f72 +testing.(*T).Run(0xc000e4b1e0, {0x4620aba, 0xc}, 0x47eae88) + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:1649 +0x856 fp=0xc00211f868 sp=0xc00211f748 pc=0x5ef6d6 +testing.runTests.func1(0x0?) + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:2054 +0x85 fp=0xc00211f8c0 sp=0xc00211f868 pc=0x5f3205 +testing.tRunner(0xc000e4b1e0, 0xc000bd1b08) + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:1595 +0x239 fp=0xc00211f9d8 sp=0xc00211f8c0 pc=0x5ede59 +testing.runTests(0xc0005ab0e0?, {0x7702460, 0x1, 0x1}, {0x1c?, 0x4aead9?, 0x0?}) + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:2052 +0x897 fp=0xc00211fb38 sp=0xc00211f9d8 pc=0x5f3077 +testing.(*M).Run(0xc0005ab0e0) + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:1925 +0xb58 fp=0xc00211feb8 sp=0xc00211fb38 pc=0x5f0898 +main.main() + _testmain.go:47 +0x2be fp=0xc00211ff40 sp=0xc00211feb8 pc=0x3e0c3fe +runtime.main() + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:267 +0x2bb fp=0xc00211ffe0 sp=0xc00211ff40 pc=0x47733b +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00211ffe8 sp=0xc00211ffe0 pc=0x4ad421 + +goroutine 2 [force gc (idle)]: +runtime.gopark(0x7541700?, 0x776b4c0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0000847a8 sp=0xc000084788 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +runtime.forcegchelper() + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:322 +0xb3 fp=0xc0000847e0 sp=0xc0000847a8 pc=0x477613 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0000847e8 sp=0xc0000847e0 pc=0x4ad421 +created by runtime.init.6 in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:310 +0x1a + +goroutine 3 [GC sweep wait]: +runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000096f78 sp=0xc000096f58 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +runtime.bgsweep(0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/mgcsweep.go:321 +0xdf fp=0xc000096fc8 sp=0xc000096f78 pc=0x4614ff +runtime.gcenable.func1() + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:200 +0x25 fp=0xc000096fe0 sp=0xc000096fc8 pc=0x4565e5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000096fe8 sp=0xc000096fe0 pc=0x4ad421 +created by runtime.gcenable in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:200 +0x66 + +goroutine 4 [GC scavenge wait]: +runtime.gopark(0xc00005a0e0?, 0x5afd9a0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000090f70 sp=0xc000090f50 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +runtime.(*scavengerState).park(0x7769d40) + /home/ec2-user/sdk/go1.21.0/src/runtime/mgcscavenge.go:425 +0x49 fp=0xc000090fa0 sp=0xc000090f70 pc=0x45ece9 +runtime.bgscavenge(0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/mgcscavenge.go:658 +0x59 fp=0xc000090fc8 sp=0xc000090fa0 pc=0x45f279 +runtime.gcenable.func2() + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:201 +0x25 fp=0xc000090fe0 sp=0xc000090fc8 pc=0x456585 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000090fe8 sp=0xc000090fe0 pc=0x4ad421 +created by runtime.gcenable in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:201 +0xa5 + +goroutine 5 [finalizer wait]: +runtime.gopark(0x0?, 0x47ef678?, 0x80?, 0x3?, 0x1000000010?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f0be28 sp=0xc001f0be08 pc=0x4777ae +runtime.runfinq() + /home/ec2-user/sdk/go1.21.0/src/runtime/mfinal.go:193 +0x13b fp=0xc001f0bfe0 sp=0xc001f0be28 pc=0x45565b +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f0bfe8 sp=0xc001f0bfe0 pc=0x4ad421 +created by runtime.createfing in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/mfinal.go:163 +0x3d + +goroutine 6 [GC worker (idle)]: +runtime.gopark(0x7c5b0aa7766c0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000183f50 sp=0xc000183f30 pc=0x4777ae +runtime.gcBgMarkWorker() + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1293 +0xe5 fp=0xc000183fe0 sp=0xc000183f50 pc=0x458165 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000183fe8 sp=0xc000183fe0 pc=0x4ad421 +created by runtime.gcBgMarkStartWorkers in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1217 +0x1c + +goroutine 18 [GC worker (idle)]: +runtime.gopark(0x7c5b0a074d332?, 0x3?, 0x44?, 0xc8?, 0xc000084fb8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000182f50 sp=0xc000182f30 pc=0x4777ae +runtime.gcBgMarkWorker() + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1293 +0xe5 fp=0xc000182fe0 sp=0xc000182f50 pc=0x458165 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000182fe8 sp=0xc000182fe0 pc=0x4ad421 +created by runtime.gcBgMarkStartWorkers in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1217 +0x1c + +goroutine 34 [GC worker (idle)]: +runtime.gopark(0x7c5b0a0746a3e?, 0x3?, 0xd4?, 0x16?, 0xc000085790?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000181f50 sp=0xc000181f30 pc=0x4777ae +runtime.gcBgMarkWorker() + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1293 +0xe5 fp=0xc000181fe0 sp=0xc000181f50 pc=0x458165 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000181fe8 sp=0xc000181fe0 pc=0x4ad421 +created by runtime.gcBgMarkStartWorkers in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1217 +0x1c + +goroutine 7 [GC worker (idle)]: +runtime.gopark(0x77a02c0?, 0x1?, 0x5c?, 0xc5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000180f50 sp=0xc000180f30 pc=0x4777ae +runtime.gcBgMarkWorker() + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1293 +0xe5 fp=0xc000180fe0 sp=0xc000180f50 pc=0x458165 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000180fe8 sp=0xc000180fe0 pc=0x4ad421 +created by runtime.gcBgMarkStartWorkers in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/runtime/mgc.go:1217 +0x1c + +goroutine 8 [select]: +runtime.gopark(0xc000092f90?, 0x2?, 0x0?, 0x0?, 0xc000092f0c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000092d80 sp=0xc000092d60 pc=0x4777ae +runtime.selectgo(0xc000092f90, 0xc000092f08, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000092ed0 sp=0xc000092d80 pc=0x48916b +github.com/golang/glog.(*fileSink).flushDaemon(0x7769918) + /home/ec2-user/go/pkg/mod/github.com/golang/glog@v1.1.1/glog_file.go:346 +0x15c fp=0xc000092fc0 sp=0xc000092ed0 pc=0x20e479c +github.com/golang/glog.init.1.func1() + /home/ec2-user/go/pkg/mod/github.com/golang/glog@v1.1.1/glog_file.go:161 +0x34 fp=0xc000092fe0 sp=0xc000092fc0 pc=0x20e2f54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000092fe8 sp=0xc000092fe0 pc=0x4ad421 +created by github.com/golang/glog.init.1 in goroutine 1 + /home/ec2-user/go/pkg/mod/github.com/golang/glog@v1.1.1/glog_file.go:161 +0x225 + +goroutine 9 [select]: +runtime.gopark(0xc000093f80?, 0x3?, 0x0?, 0x0?, 0xc000093f1a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000093d90 sp=0xc000093d70 pc=0x4777ae +runtime.selectgo(0xc000093f80, 0xc000093f14, 0xc000436680?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000093ee0 sp=0xc000093d90 pc=0x48916b +go.opencensus.io/stats/view.(*worker).start(0xc000436680) + /home/ec2-user/go/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:292 +0x199 fp=0xc000093fc0 sp=0xc000093ee0 pc=0x3bf4f39 +go.opencensus.io/stats/view.init.0.func1() + /home/ec2-user/go/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:34 +0x34 fp=0xc000093fe0 sp=0xc000093fc0 pc=0x3bf39b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000093fe8 sp=0xc000093fe0 pc=0x4ad421 +created by go.opencensus.io/stats/view.init.0 in goroutine 1 + /home/ec2-user/go/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:34 +0xfd + +goroutine 117 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xf0?, 0xa1?, 0xc000187f98?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000187f08 sp=0xc000187ee8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000187f80 sp=0xc000187f08 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0x7f429a401500?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc000187fa8 sp=0xc000187f80 pc=0x440f72 +github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore.func3() + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:99 +0x4a fp=0xc000187fe0 sp=0xc000187fa8 pc=0x24a06aa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000187fe8 sp=0xc000187fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore in goroutine 65 + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:98 +0x4cb + +goroutine 87 [IO wait]: +runtime.gopark(0x4?, 0xc000091c30?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000091c48 sp=0xc000091c28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000091c80 sp=0xc000091c48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6eee0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000091ca0 sp=0xc000091c80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000dd7100, 0xc000de2000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000091ce8 sp=0xc000091ca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000dd70e0, {0xc000de2000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000091dd8 sp=0xc000091ce8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc00007d670, {0xc000de2000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc000091e38 sp=0xc000091dd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc0008621b0}, {0x5b0eb40, 0xc00007d670}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc000091ee0 sp=0xc000091e38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc000091f50 sp=0xc000091ee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc000996600) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc000091fb8 sp=0xc000091f50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc000091fe0 sp=0xc000091fb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000091fe8 sp=0xc000091fe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 65 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 116 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc0007d2640?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0008b4f08 sp=0xc0008b4ee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc000ef4250, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc0008b4f58 sp=0xc0008b4f08 pc=0x4a9339 +sync.(*Cond).Wait(0xc000ef4240) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc0008b4f88 sp=0xc0008b4f58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc0007d2640) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc0008b4fc0 sp=0xc0008b4f88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc0008b4fe0 sp=0xc0008b4fc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008b4fe8 sp=0xc0008b4fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 65 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 86 [select, locked to thread]: +runtime.gopark(0xc000095fa8?, 0x2?, 0x0?, 0x0?, 0xc000095fa4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000095e08 sp=0xc000095de8 pc=0x4777ae +runtime.selectgo(0xc000095fa8, 0xc000095fa0, 0x0?, 0x0, 0xc0008ab150?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000095f58 sp=0xc000095e08 pc=0x48916b +runtime.ensureSigM.func1() + /home/ec2-user/sdk/go1.21.0/src/runtime/signal_unix.go:1014 +0x19f fp=0xc000095fe0 sp=0xc000095f58 pc=0x4a35df +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000095fe8 sp=0xc000095fe0 pc=0x4ad421 +created by runtime.ensureSigM in goroutine 85 + /home/ec2-user/sdk/go1.21.0/src/runtime/signal_unix.go:997 +0xc8 + +goroutine 85 [select]: +runtime.gopark(0xc00105f408?, 0x5?, 0x10?, 0xea?, 0xc00105eefe?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00105e9c0 sp=0xc00105e9a0 pc=0x4777ae +runtime.selectgo(0xc00105f408, 0xc00105eef4, 0x7769700?, 0x0, 0x710b428a?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00105eb10 sp=0xc00105e9c0 pc=0x48916b +github.com/onsi/ginkgo/v2/internal.(*Suite).runNode(_, {0x39, 0x4, {0x468d28e, 0x29}, 0xc0001255c0, {{0x64228e0, 0x35}, 0x10d, {0x0, ...}, ...}, ...}, ...) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/suite.go:887 +0x182f fp=0xc001062a40 sp=0xc00105eb10 pc=0xb9372f +github.com/onsi/ginkgo/v2/internal.(*group).attemptSpec(0xc001066830, 0x1, {{0xc000f9a480?, 0xc0030d2ee0?, 0x2?}, 0x0?}) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/group.go:199 +0x1125 fp=0xc001064ab0 sp=0xc001062a40 pc=0xb68705 +github.com/onsi/ginkgo/v2/internal.(*group).run(0xc001066830, {0xc0017df6c0, 0xd, 0xd}) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/group.go:349 +0x1228 fp=0xc001066498 sp=0xc001064ab0 pc=0xb6c0e8 +github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000ca4000, {0x462619f, 0xe}, {0x779c980, 0x0, 0x0}, {0xc000060064, 0x29}, 0x0, {0xc000ac3800, ...}) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/suite.go:465 +0x1167 fp=0xc001067270 sp=0xc001066498 pc=0xb8bc07 +github.com/onsi/ginkgo/v2/internal.(*Suite).Run(_, {_, _}, {_, _, _}, {_, _}, _, {0x5b4ff78, ...}, ...) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/suite.go:116 +0x5f8 fp=0xc001067438 sp=0xc001067270 pc=0xb85518 +github.com/onsi/ginkgo/v2.RunSpecs({0x5b0ed20, 0xc000e4b380}, {0x462619f, 0xe}, {0x0, 0x0, 0x0?}) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/core_dsl.go:319 +0x166b fp=0xc001067e48 sp=0xc001067438 pc=0xba34cb +github.com/rancher/opni/test/plugins/alerting_test.TestAlerting(0x0?) + /home/ec2-user/opni/test/plugins/alerting/alerting_suite_test.go:40 +0x4e fp=0xc001067e98 sp=0xc001067e48 pc=0x3dc86ee +testing.tRunner(0xc000e4b380, 0x47eae88) + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:1595 +0x239 fp=0xc001067fb0 sp=0xc001067e98 pc=0x5ede59 +testing.(*T).Run.func1() + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:1648 +0x45 fp=0xc001067fe0 sp=0xc001067fb0 pc=0x5ef945 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001067fe8 sp=0xc001067fe0 pc=0x4ad421 +created by testing.(*T).Run in goroutine 1 + /home/ec2-user/sdk/go1.21.0/src/testing/testing.go:1648 +0x82b + +goroutine 62 [syscall]: +runtime.notetsleepg(0x4b0571?, 0x4ad421?) + /home/ec2-user/sdk/go1.21.0/src/runtime/lock_futex.go:236 +0x29 fp=0xc000084fa0 sp=0xc000084f68 pc=0x447049 +os/signal.signal_recv() + /home/ec2-user/sdk/go1.21.0/src/runtime/sigqueue.go:152 +0x29 fp=0xc000084fc0 sp=0xc000084fa0 pc=0x4a9729 +os/signal.loop() + /home/ec2-user/sdk/go1.21.0/src/os/signal/signal_unix.go:23 +0x1d fp=0xc000084fe0 sp=0xc000084fc0 pc=0x6a909d +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000084fe8 sp=0xc000084fe0 pc=0x4ad421 +created by os/signal.Notify.func1.1 in goroutine 85 + /home/ec2-user/sdk/go1.21.0/src/os/signal/signal.go:151 +0x47 + +goroutine 63 [select]: +runtime.gopark(0xc000185f78?, 0x3?, 0x0?, 0x0?, 0xc000185f22?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000185d88 sp=0xc000185d68 pc=0x4777ae +runtime.selectgo(0xc000185f78, 0xc000185f1c, 0xc000185f28?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000185ed8 sp=0xc000185d88 pc=0x48916b +github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0x0) + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/interrupt_handler/interrupt_handler.go:131 +0x125 fp=0xc000185fb8 sp=0xc000185ed8 pc=0xb57045 +github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func3() + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/interrupt_handler/interrupt_handler.go:158 +0x42 fp=0xc000185fe0 sp=0xc000185fb8 pc=0xb56ee2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000185fe8 sp=0xc000185fe0 pc=0x4ad421 +created by github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 85 + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/interrupt_handler/interrupt_handler.go:128 +0x2bd + +goroutine 64 [select]: +runtime.gopark(0xc001087fb0?, 0x2?, 0xff?, 0xff?, 0xc001087f7c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001087df0 sp=0xc001087dd0 pc=0x4777ae +runtime.selectgo(0xc001087fb0, 0xc001087f78, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001087f40 sp=0xc001087df0 pc=0x48916b +github.com/onsi/ginkgo/v2/internal.RegisterForProgressSignal.func1() + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/progress_report.go:32 +0xc7 fp=0xc001087fe0 sp=0xc001087f40 pc=0xb7f707 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001087fe8 sp=0xc001087fe0 pc=0x4ad421 +created by github.com/onsi/ginkgo/v2/internal.RegisterForProgressSignal in goroutine 85 + /home/ec2-user/go/pkg/mod/github.com/onsi/ginkgo/v2@v2.12.0/internal/progress_report.go:30 +0x189 + +goroutine 636 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f419a8 sp=0xc001f41988 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001f419e0 sp=0xc001f419a8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b298, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001f41a00 sp=0xc001f419e0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001c9bc20, 0x448801?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001f41a48 sp=0xc001f41a00 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc001c9bc00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001f41b38 sp=0xc001f41a48 pc=0x513f65 +net.(*netFD).accept(0xc001c9bc00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001f41c30 sp=0xc001f41b38 pc=0x8967be +net.(*TCPListener).accept(0xc0007d2a60) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc001f41c78 sp=0xc001f41c30 pc=0x8c383e +net.(*TCPListener).Accept(0xc0007d2a60) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc001f41ce8 sp=0xc001f41c78 pc=0x8c1ba5 +net/http.(*onceCloseListener).Accept(0xc000e38810) + :1 +0x44 fp=0xc001f41d20 sp=0xc001f41ce8 pc=0xab5544 +net/http.(*Server).Serve(0xc001ec2780, {0x5b2a6d0, 0xc0007d2a60}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc001f41f18 sp=0xc001f41d20 pc=0xa7859f +net/http.(*Server).ListenAndServe(0xc001ec2780) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2985 +0xbd fp=0xc001f41f70 sp=0xc001f41f18 pc=0xa77f9d +github.com/rancher/opni/pkg/management.(*Server).listenAndServeHttp.func2() + /home/ec2-user/opni/pkg/management/server.go:246 +0x2f fp=0xc001f41fa0 sp=0xc001f41f70 pc=0x231edef +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc001f41fe0 sp=0xc001f41fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f41fe8 sp=0xc001f41fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 496 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 88 [IO wait]: +runtime.gopark(0x4b05a9?, 0xc0008aec70?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0008aec48 sp=0xc0008aec28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0008aec80 sp=0xc0008aec48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6ecf0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0008aeca0 sp=0xc0008aec80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000dd71c0, 0xc000128000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0008aece8 sp=0xc0008aeca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000dd71a0, {0xc000128000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0008aedd8 sp=0xc0008aece8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc00007d688, {0xc000128000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc0008aee38 sp=0xc0008aedd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc0008621c8}, {0x5b0eb40, 0xc00007d688}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc0008aeee0 sp=0xc0008aee38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc0008aef50 sp=0xc0008aeee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc000996640) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc0008aefb8 sp=0xc0008aef50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc0008aefe0 sp=0xc0008aefb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008aefe8 sp=0xc0008aefe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 65 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 89 [select]: +runtime.gopark(0xc0009822b0?, 0x656a30?, 0x0?, 0x1?, 0xc0008b2e7c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0008b2ce0 sp=0xc0008b2cc0 pc=0x4777ae +runtime.selectgo(0xc0008b2f60, 0xc0008b2e78, 0xc0009b9168?, 0x1, 0xc0009c5bd0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0008b2e30 sp=0xc0008b2ce0 pc=0x48916b +os/exec.(*Cmd).watchCtx(0xc000b32000, 0xc0009822a0) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:757 +0x10f fp=0xc0008b2fb0 sp=0xc0008b2e30 pc=0x656a6f +os/exec.(*Cmd).Start.func4() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0x45 fp=0xc0008b2fe0 sp=0xc0008b2fb0 pc=0x6565a5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008b2fe8 sp=0xc0008b2fe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 65 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0xf05 + +goroutine 90 [syscall]: +syscall.Syscall6(0x8200000000?, 0x82000000c7?, 0x20000000c5?, 0x1ef?, 0x64?, 0x1ef?, 0x63?) + /home/ec2-user/sdk/go1.21.0/src/syscall/syscall_linux.go:91 +0x3d fp=0xc0008b3d38 sp=0xc0008b3cb0 pc=0x4d1b1d +os.(*Process).blockUntilWaitable(0xc0009a8f00) + /home/ec2-user/sdk/go1.21.0/src/os/wait_waitid.go:32 +0x9f fp=0xc0008b3e18 sp=0xc0008b3d38 pc=0x529c5f +os.(*Process).wait(0xc0009a8f00) + /home/ec2-user/sdk/go1.21.0/src/os/exec_unix.go:22 +0x4b fp=0xc0008b3e98 sp=0xc0008b3e18 pc=0x51e0ab +os.(*Process).Wait(...) + /home/ec2-user/sdk/go1.21.0/src/os/exec.go:134 +os/exec.(*Cmd).Wait(0xc000b32000) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:890 +0xb9 fp=0xc0008b3f60 sp=0xc0008b3e98 pc=0x657279 +github.com/onsi/gomega/gexec.(*Session).monitorForExit(0xc000ddd140, 0x5b21760?) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:196 +0x3f fp=0xc0008b3fb0 sp=0xc0008b3f60 pc=0x233731f +github.com/onsi/gomega/gexec.Start.func1() + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x45 fp=0xc0008b3fe0 sp=0xc0008b3fb0 pc=0x2336d65 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008b3fe8 sp=0xc0008b3fe0 pc=0x4ad421 +created by github.com/onsi/gomega/gexec.Start in goroutine 65 + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x9eb + +goroutine 91 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x8c?, 0xed?, 0x446b50?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001084e78 sp=0xc001084e58 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001084ef0 sp=0xc001084e78 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0xc000085f80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001084f18 sp=0xc001084ef0 pc=0x440f72 +github.com/rancher/opni/pkg/test.(*Environment).startJetstream.func1() + /home/ec2-user/opni/pkg/test/environment.go:562 +0x69 fp=0xc001084f50 sp=0xc001084f18 pc=0x24b23e9 +github.com/rancher/opni/pkg/util/waitctx.restrictive.Go.func1() + /home/ec2-user/opni/pkg/util/waitctx/context.go:162 +0x93 fp=0xc001084fe0 sp=0xc001084f50 pc=0x178df73 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001084fe8 sp=0xc001084fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/util/waitctx.restrictive.Go in goroutine 65 + /home/ec2-user/opni/pkg/util/waitctx/context.go:160 +0xeb + +goroutine 100 [select]: +runtime.gopark(0xc000d03f48?, 0x2?, 0xb2?, 0x0?, 0xc000d03f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d03d80 sp=0xc000d03d60 pc=0x4777ae +runtime.selectgo(0xc000d03f48, 0xc000d03f00, 0x104000000b2?, 0x0, 0x144000000b2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d03ed0 sp=0xc000d03d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).Recv(0xc000fe6b80) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:234 +0xeb fp=0xc000d03f78 sp=0xc000d03ed0 pc=0x2155d2b +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc000ee4410) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc000d03fc0 sp=0xc000d03f78 pc=0x215759f +github.com/hashicorp/go-plugin.newGRPCClient.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x34 fp=0xc000d03fe0 sp=0xc000d03fc0 pc=0x21599b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d03fe8 sp=0xc000d03fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 154 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x5b7 + +goroutine 115 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0x28?, 0x71?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001081e18 sp=0xc001081df8 pc=0x4777ae +runtime.chanrecv(0xc000066480, 0xc001081ed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001081e90 sp=0xc001081e18 pc=0x441325 +runtime.chanrecv2(0xc000c77128?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc001081eb8 sp=0xc001081e90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc000c77100) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc001081fc0 sp=0xc001081eb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc001081fe0 sp=0xc001081fc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001081fe8 sp=0xc001081fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 65 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 114 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x26?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001080bd0 sp=0xc001080bb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001080c08 sp=0xc001080bd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6ebf8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001080c28 sp=0xc001080c08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000436220, 0xc000df6000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001080c70 sp=0xc001080c28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000436200, {0xc000df6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001080d60 sp=0xc001080c70 pc=0x50e5a5 +net.(*netFD).Read(0xc000436200, {0xc000df6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001080dc0 sp=0xc001080d60 pc=0x89342b +net.(*conn).Read(0xc000b780d8, {0xc000df6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001080e48 sp=0xc001080dc0 pc=0x8b176d +net.(*TCPConn).Read(0xc000b780d8, {0xc000df6000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc001080e90 sp=0xc001080e48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc000ef4b00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc001080f10 sp=0xc001080e90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc000c77100) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc001080fc0 sp=0xc001080f10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc001080fe0 sp=0xc001080fc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001080fe8 sp=0xc001080fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 65 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 96 [chan receive]: +runtime.gopark(0x0?, 0x10?, 0xa0?, 0xf0?, 0xc0027a51e4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001082d10 sp=0xc001082cf0 pc=0x4777ae +runtime.chanrecv(0xc0008abe00, 0xc001082e40, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001082d88 sp=0xc001082d10 pc=0x441325 +runtime.chanrecv2(0x3b9aca00?, 0x7f429a406300?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc001082db0 sp=0xc001082d88 pc=0x440f92 +github.com/hashicorp/go-plugin.pidWait(0xc000d74200?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/process.go:20 +0xdb fp=0xc001082e68 sp=0xc001082db0 pc=0x216033b +github.com/hashicorp/go-plugin.(*Client).reattach.func1(0x0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:853 +0xd8 fp=0xc001082fb8 sp=0xc001082e68 pc=0x21528b8 +github.com/hashicorp/go-plugin.(*Client).reattach.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:862 +0x42 fp=0xc001082fe0 sp=0xc001082fb8 pc=0x21527a2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001082fe8 sp=0xc001082fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*Client).reattach in goroutine 154 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:846 +0x365 + +goroutine 118 [select]: +runtime.gopark(0xc000066730?, 0x13b0513?, 0x0?, 0x1?, 0xc0008afecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0008afd30 sp=0xc0008afd10 pc=0x4777ae +runtime.selectgo(0xc0008aff60, 0xc0008afec8, 0x446b50?, 0x1, 0xc000405798?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0008afe80 sp=0xc0008afd30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc000c95860) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:64 +0x318 fp=0xc0008affc0 sp=0xc0008afe80 pc=0x13b0538 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc0008affe0 sp=0xc0008affc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008affe8 sp=0xc0008affe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 65 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 28 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc000ac80d0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f6ddd8 sp=0xc000f6ddb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc0003f8410, 0x19) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc000f6de28 sp=0xc000f6ddd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc0003f8400) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc000f6de58 sp=0xc000f6de28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc000c77100, 0xc000ac80d0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc000f6dfb0 sp=0xc000f6de58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc000f6dfe0 sp=0xc000f6dfb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f6dfe8 sp=0xc000f6dfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 65 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 125 [select]: +runtime.gopark(0xc0008b5e40?, 0x3?, 0xb4?, 0x1b?, 0xc0008b5c2a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0008b5a48 sp=0xc0008b5a28 pc=0x4777ae +runtime.selectgo(0xc0008b5e40, 0xc0008b5c24, 0x0?, 0x0, 0xc0009e77c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0008b5b98 sp=0xc0008b5a48 pc=0x48916b +github.com/rancher/opni/pkg/health.(*Monitor).Run(0xc00106e820, {0x5b426e0, 0xc000b69f40}, {0x5b21aa8, 0xc000436580}) + /home/ec2-user/opni/pkg/health/monitor.go:69 +0x1f3 fp=0xc0008b5f80 sp=0xc0008b5b98 pc=0x206e173 +github.com/rancher/opni/pkg/gateway.NewGateway.func23() + /home/ec2-user/opni/pkg/gateway/gateway.go:277 +0x6c fp=0xc0008b5fe0 sp=0xc0008b5f80 pc=0x22aed8c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008b5fe8 sp=0xc0008b5fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.NewGateway in goroutine 65 + /home/ec2-user/opni/pkg/gateway/gateway.go:277 +0x3eed + +goroutine 596 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020e19e0 sp=0xc0020e19c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0020e1a18 sp=0xc0020e19e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a870, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0020e1a38 sp=0xc0020e1a18 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0019e7e20, 0x4b0501?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0020e1a80 sp=0xc0020e1a38 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0019e7e00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0020e1b70 sp=0xc0020e1a80 pc=0x513f65 +net.(*netFD).accept(0xc0019e7e00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0020e1c68 sp=0xc0020e1b70 pc=0x8967be +net.(*TCPListener).accept(0xc001f0e380) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc0020e1cb0 sp=0xc0020e1c68 pc=0x8c383e +net.(*TCPListener).Accept(0xc001f0e380) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc0020e1d20 sp=0xc0020e1cb0 pc=0x8c1ba5 +google.golang.org/grpc.(*Server).Serve(0xc000e0c3c0, {0x5b2a6d0, 0xc001f0e380}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0020e1f50 sp=0xc0020e1d20 pc=0x117789e +github.com/rancher/opni/pkg/management.(*Server).listenAndServeGrpc.func1() + /home/ec2-user/opni/pkg/management/server.go:213 +0x5e fp=0xc0020e1fa0 sp=0xc0020e1f50 pc=0x231e69e +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0020e1fe0 sp=0xc0020e1fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020e1fe8 sp=0xc0020e1fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 495 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 126 [chan receive]: +runtime.gopark(0x0?, 0x4b05a9?, 0x50?, 0x5e?, 0x4bff4a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001085e18 sp=0xc001085df8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001085e90 sp=0xc001085e18 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0x7f429a405700?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001085eb8 sp=0xc001085e90 pc=0x440f72 +github.com/rancher/opni/pkg/gateway.NewGateway.func5() + /home/ec2-user/opni/pkg/gateway/gateway.go:308 +0x5c fp=0xc001085f50 sp=0xc001085eb8 pc=0x22aea9c +github.com/rancher/opni/pkg/util/waitctx.restrictive.Go.func1() + /home/ec2-user/opni/pkg/util/waitctx/context.go:162 +0x93 fp=0xc001085fe0 sp=0xc001085f50 pc=0x178df73 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001085fe8 sp=0xc001085fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/util/waitctx.restrictive.Go in goroutine 65 + /home/ec2-user/opni/pkg/util/waitctx/context.go:160 +0xeb + +goroutine 553 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bc48d8 sp=0xc001bc48b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001bc4910 sp=0xc001bc48d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677bc8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001bc4930 sp=0xc001bc4910 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d73920, 0xc001de8601?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001bc4978 sp=0xc001bc4930 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000d73900) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001bc4a68 sp=0xc001bc4978 pc=0x513f65 +net.(*netFD).accept(0xc000d73900) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001bc4b60 sp=0xc001bc4a68 pc=0x8967be +net.(*TCPListener).accept(0xc000997980) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc001bc4ba8 sp=0xc001bc4b60 pc=0x8c383e +net.(*TCPListener).Accept(0xc000997980) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc001bc4c18 sp=0xc001bc4ba8 pc=0x8c1ba5 +crypto/tls.(*listener).Accept(0xc000863bd8) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/tls.go:66 +0x43 fp=0xc001bc4c70 sp=0xc001bc4c18 pc=0x9b2d43 +net/http.(*onceCloseListener).Accept(0xc001dcf740) + :1 +0x44 fp=0xc001bc4ca8 sp=0xc001bc4c70 pc=0xab5544 +net/http.(*Server).Serve(0xc001da51d0, {0x5b2a5b0, 0xc000863bd8}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc001bc4ea0 sp=0xc001bc4ca8 pc=0xa7859f +net/http.Serve(...) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2595 +github.com/gin-gonic/gin.(*Engine).RunListener(0xc000c4c820, {0x5b2a5b0, 0xc000863bd8}) + /home/ec2-user/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/gin.go:565 +0x370 fp=0xc001bc4f90 sp=0xc001bc4ea0 pc=0x171e870 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure.func1() + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:72 +0x5c fp=0xc001bc4fe0 sp=0xc001bc4f90 pc=0x262a57c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc4fe8 sp=0xc001bc4fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure in goroutine 176 + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:71 +0x26d + +goroutine 573 [select]: +runtime.gopark(0xc001f09478?, 0x2?, 0xe8?, 0x28?, 0xc001f09414?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f09280 sp=0xc001f09260 pc=0x4777ae +runtime.selectgo(0xc001f09478, 0xc001f09410, 0xc001dcf980?, 0x0, 0xc001f09470?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f093d0 sp=0xc001f09280 pc=0x48916b +github.com/rancher/opni/pkg/management.(*Server).WatchClusterHealthStatus(0xc000d8a000, 0x42219c0?, {0x5b4cae0, 0xc001dc4ed0}) + /home/ec2-user/opni/pkg/management/health.go:38 +0x352 fp=0xc001f094a8 sp=0xc001f093d0 pc=0x23189f2 +github.com/rancher/opni/pkg/apis/management/v1._Management_WatchClusterHealthStatus_Handler({0x4537040?, 0xc000d8a000}, {0x5b4ab30, 0xc001dcf9b0}) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:829 +0xfa fp=0xc001f09508 sp=0xc001f094a8 pc=0x1781ada +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4537040, 0xc000d8a000}, {0x5b492f8, 0xc001da52c0}, 0xc00149ffb0, 0x47ec6b8) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001f09848 sp=0xc001f09508 pc=0x12b0e02 +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef860, {0x5b54e00, 0xc001174680}, 0xc001f18240, 0xc000ff8960, 0x7716c80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001f09db0 sp=0xc001f09848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef860, {0x5b54e00, 0xc001174680}, 0xc001f18240, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001f09f28 sp=0xc001f09db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001f09fe0 sp=0xc001f09f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f09fe8 sp=0xc001f09fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 371 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 629 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000184ee8 sp=0xc000184ec8 pc=0x4777ae +runtime.chanrecv(0xc000f26540, 0xc000184fa0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000184f60 sp=0xc000184ee8 pc=0x441325 +runtime.chanrecv2(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc000184f88 sp=0xc000184f60 pc=0x440f92 +google.golang.org/grpc.(*Server).serverWorker(0xc001ebf4a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:593 +0x85 fp=0xc000184fc0 sp=0xc000184f88 pc=0x11744a5 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc000184fe0 sp=0xc000184fc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000184fe8 sp=0xc000184fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 559 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 565 [select]: +runtime.gopark(0xc001ddd990?, 0x2?, 0x18?, 0xd8?, 0xc001ddd944?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ddd7c0 sp=0xc001ddd7a0 pc=0x4777ae +runtime.selectgo(0xc001ddd990, 0xc001ddd940, 0x4b0b59?, 0x0, 0x100000001?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ddd910 sp=0xc001ddd7c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0019cbd40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001ddd9c0 sp=0xc001ddd910 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bf5ee0, {0x432c3a0?, 0xc0017e0600}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc001dddc20 sp=0xc001ddd9c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0019cbbb4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001dddc80 sp=0xc001dddc20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0019cbb00, 0xc001ddde38, 0xc001ddde08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc001dddd28 sp=0xc001dddc80 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019cbb00, {0x432c3a0, 0xc0017e0600}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001ddde68 sp=0xc001dddd28 pc=0x119077e +github.com/rancher/opni/pkg/apis/management/v1.(*managementWatchClustersClient).Recv(0xc001bf11a0) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:184 +0x66 fp=0xc001dddea8 sp=0xc001ddde68 pc=0x177c726 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).watchGlobalCluster(0xc0005a8c80, {0x5b6f168, 0xc000fe1300}, 0xc000b529b0?) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/management.go:124 +0x1de fp=0xc001dddf70 sp=0xc001dddea8 pc=0x396477e +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).useWatchers.func1() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:246 +0x55 fp=0xc001dddfc0 sp=0xc001dddf70 pc=0x3973035 +github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents.func1() + /home/ec2-user/opni/pkg/management/watcher.go:27 +0x2f fp=0xc001dddfe0 sp=0xc001dddfc0 pc=0x23220cf +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001dddfe8 sp=0xc001dddfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents in goroutine 177 + /home/ec2-user/opni/pkg/management/watcher.go:26 +0x54 + +goroutine 630 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010e99e0 sp=0xc0010e99c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0010e9a18 sp=0xc0010e99e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a588, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0010e9a38 sp=0xc0010e9a18 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001651920, 0x592701?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0010e9a80 sp=0xc0010e9a38 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc001651900) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0010e9b70 sp=0xc0010e9a80 pc=0x513f65 +net.(*netFD).accept(0xc001651900) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0010e9c68 sp=0xc0010e9b70 pc=0x8967be +net.(*TCPListener).accept(0xc001fec100) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc0010e9cb0 sp=0xc0010e9c68 pc=0x8c383e +net.(*TCPListener).Accept(0xc001fec100) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc0010e9d20 sp=0xc0010e9cb0 pc=0x8c1ba5 +google.golang.org/grpc.(*Server).Serve(0xc001ebf4a0, {0x5b2a6d0, 0xc001fec100}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0010e9f50 sp=0xc0010e9d20 pc=0x117789e +github.com/rancher/opni/pkg/gateway.(*GatewayGRPCServer).ListenAndServe.func1() + /home/ec2-user/opni/pkg/gateway/grpc.go:105 +0x47 fp=0xc0010e9fa0 sp=0xc0010e9f50 pc=0x22b4c27 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0010e9fe0 sp=0xc0010e9fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010e9fe8 sp=0xc0010e9fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 559 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 149 [select]: +runtime.gopark(0xc000f69da8?, 0x2?, 0x60?, 0x98?, 0xc000f69a94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f69818 sp=0xc000f697f8 pc=0x4777ae +runtime.selectgo(0xc000f69da8, 0xc000f69a90, 0x0?, 0x0, 0x1860a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f69968 sp=0xc000f69818 pc=0x48916b +github.com/hashicorp/go-plugin.Serve(0xc000bc3e00) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:474 +0x2091 fp=0xc000f69fc0 sp=0xc000f69968 pc=0x21665b1 +github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins.func4() + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x34 fp=0xc000f69fe0 sp=0xc000f69fc0 pc=0x24cd234 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f69fe8 sp=0xc000f69fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins in goroutine 65 + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x76b + +goroutine 150 [select]: +runtime.gopark(0xc001083f48?, 0x2?, 0x20?, 0x3f?, 0xc001083f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001083d80 sp=0xc001083d60 pc=0x4777ae +runtime.selectgo(0xc001083f48, 0xc001083f00, 0x0?, 0x0, 0x20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001083ed0 sp=0xc001083d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).Recv(0xc000c4a630) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:124 +0xeb fp=0xc001083f78 sp=0xc001083ed0 pc=0x2154feb +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc000451540) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc001083fc0 sp=0xc001083f78 pc=0x215759f +github.com/hashicorp/go-plugin.(*GRPCServer).Init.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x34 fp=0xc001083fe0 sp=0xc001083fc0 pc=0x215adf4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001083fe8 sp=0xc001083fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*GRPCServer).Init in goroutine 149 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x89d + +goroutine 479 [select]: +runtime.gopark(0xc001587e70?, 0x2?, 0x0?, 0x0?, 0xc001587df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002239c58 sp=0xc002239c38 pc=0x4777ae +runtime.selectgo(0xc002239e70, 0xc001587df0, 0xc001587e00?, 0x0, 0xc0010b8000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002239da8 sp=0xc002239c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139d4f0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002239ea0 sp=0xc002239da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000611880) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002239f58 sp=0xc002239ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc002239fe0 sp=0xc002239f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002239fe8 sp=0xc002239fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 392 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 689 [chan receive]: +runtime.gopark(0x110a2c12646e656b?, 0x43646e656b636162?, 0x6f?, 0x6d?, 0x5209280320011873?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d3d60 sp=0xc0017d3d40 pc=0x4777ae +runtime.chanrecv(0xc0000672c0, 0xc0017d3e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0017d3dd8 sp=0xc0017d3d60 pc=0x441325 +runtime.chanrecv1(0x696e6f4d6e6f6974?, 0x6142676e69726f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0017d3e00 sp=0xc0017d3dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0017d3fe0 sp=0xc0017d3e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d3fe8 sp=0xc0017d3fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 153 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a6f940 sp=0xc000a6f920 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000a6f978 sp=0xc000a6f940 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6ede8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000a6f998 sp=0xc000a6f978 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000c42920, 0xc000a6fa01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000a6f9e0 sp=0xc000a6f998 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000c42900) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc000a6fad0 sp=0xc000a6f9e0 pc=0x513f65 +net.(*netFD).accept(0xc000c42900) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc000a6fbc8 sp=0xc000a6fad0 pc=0x8967be +net.(*UnixListener).accept(0xc000c4a270) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc000a6fc08 sp=0xc000a6fbc8 pc=0x8cedfe +net.(*UnixListener).Accept(0xc000c4a270) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc000a6fc78 sp=0xc000a6fc08 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc000b74b00) + :1 +0x44 fp=0xc000a6fcb0 sp=0xc000a6fc78 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc000c401e0, {0x5b2c6a0, 0xc000b74b00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc000a6fee0 sp=0xc000a6fcb0 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCServer).Serve(0xc000bbcc60, {0x5b2c6a0, 0xc000b74b00}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:153 +0xdd fp=0xc000a6ff98 sp=0xc000a6fee0 pc=0x215b33d +github.com/hashicorp/go-plugin.Serve.func6() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x62 fp=0xc000a6ffe0 sp=0xc000a6ff98 pc=0x2166b62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a6ffe8 sp=0xc000a6ffe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.Serve in goroutine 149 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x1f67 + +goroutine 1127 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011a28d8 sp=0xc0011a28b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0011a2910 sp=0xc0011a28d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163bc30, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0011a2930 sp=0xc0011a2910 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193b220, 0xc001838001?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0011a2978 sp=0xc0011a2930 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00193b200) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0011a2a68 sp=0xc0011a2978 pc=0x513f65 +net.(*netFD).accept(0xc00193b200) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0011a2b60 sp=0xc0011a2a68 pc=0x8967be +net.(*TCPListener).accept(0xc000b30e00) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc0011a2ba8 sp=0xc0011a2b60 pc=0x8c383e +net.(*TCPListener).Accept(0xc000b30e00) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc0011a2c18 sp=0xc0011a2ba8 pc=0x8c1ba5 +crypto/tls.(*listener).Accept(0xc000126f90) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/tls.go:66 +0x43 fp=0xc0011a2c70 sp=0xc0011a2c18 pc=0x9b2d43 +net/http.(*onceCloseListener).Accept(0xc00294ac60) + :1 +0x44 fp=0xc0011a2ca8 sp=0xc0011a2c70 pc=0xab5544 +net/http.(*Server).Serve(0xc001344e10, {0x5b2a5b0, 0xc000126f90}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc0011a2ea0 sp=0xc0011a2ca8 pc=0xa7859f +net/http.Serve(...) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2595 +github.com/gin-gonic/gin.(*Engine).RunListener(0xc00190eea0, {0x5b2a5b0, 0xc000126f90}) + /home/ec2-user/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/gin.go:565 +0x370 fp=0xc0011a2f90 sp=0xc0011a2ea0 pc=0x171e870 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure.func1() + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:72 +0x5c fp=0xc0011a2fe0 sp=0xc0011a2f90 pc=0x262a57c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011a2fe8 sp=0xc0011a2fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure in goroutine 848 + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:71 +0x26d + +goroutine 542 [select]: +runtime.gopark(0xc001ba3f70?, 0x2?, 0xb4?, 0x1b?, 0xc001ba3ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ba3d20 sp=0xc001ba3d00 pc=0x4777ae +runtime.selectgo(0xc001ba3f70, 0xc001ba3eb8, 0xc0009d3fe0?, 0x0, 0xc0005a8e30?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ba3e70 sp=0xc001ba3d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000ee7400, {0x5b426e0, 0xc00010f680}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001ba3fa0 sp=0xc001ba3e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001ba3fe0 sp=0xc001ba3fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ba3fe8 sp=0xc001ba3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 434 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 527 [select]: +runtime.gopark(0xc000a51f70?, 0x4?, 0x54?, 0xb6?, 0xc000a51dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a51c00 sp=0xc000a51be0 pc=0x4777ae +runtime.selectgo(0xc000a51f70, 0xc000a51dd0, 0x3920159?, 0x0, 0xc000931c00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a51d50 sp=0xc000a51c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0019e4820) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc000a51fc0 sp=0xc000a51d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc000a51fe0 sp=0xc000a51fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a51fe8 sp=0xc000a51fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 501 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 458 [select]: +runtime.gopark(0xc001169448?, 0x2?, 0xd0?, 0x92?, 0xc0011693fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001169278 sp=0xc001169258 pc=0x4777ae +runtime.selectgo(0xc001169448, 0xc0011693f8, 0xc000813988?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011693c8 sp=0xc001169278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0010dcc60) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001169478 sp=0xc0011693c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001676750, {0x42219c0?, 0xc000ff8b40}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0011696d8 sp=0xc001169478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0010dc894?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001169738 sp=0xc0011696d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0010dc7e0, 0xc0011698f0, 0xc0011698c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc0011697e0 sp=0xc001169738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010dc7e0, {0x42219c0, 0xc000ff8b40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001169920 sp=0xc0011697e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc000ff8cc0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc000ff8a50}, {0x42219c0, 0xc000ff8b40}, 0x0?, {0xc000996a60, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc0011699c0 sp=0xc001169920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc000762460}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc000ff8a50}, {0x42219c0, 0xc000ff8b40}, 0xc000f7a000, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc001169d30 sp=0xc0011699c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc000f7a000, {0x5b426e0, 0xc000762460}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc000ff8a50}, {0x42219c0, 0xc000ff8b40}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc001169e58 sp=0xc001169d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseKeyValueStore(0xc00158aa70, {0x5b426e0, 0xc000762460}, 0x0?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:69 +0xee fp=0xc001169ee8 sp=0xc001169e58 pc=0x21ab96e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore.func2(0x3) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:219 +0xf2 fp=0xc001169f58 sp=0xc001169ee8 pc=0x21a85d2 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc001169fe0 sp=0xc001169f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001169fe8 sp=0xc001169fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 456 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 1191 [select]: +runtime.gopark(0xc00189be70?, 0x2?, 0x0?, 0x0?, 0xc00189bdf4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00189bc58 sp=0xc00189bc38 pc=0x4777ae +runtime.selectgo(0xc00189be70, 0xc00189bdf0, 0xc00189be00?, 0x0, 0xc001ce3b58?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00189bda8 sp=0xc00189bc58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0015fbea0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00189bea0 sp=0xc00189bda8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0014fae70) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00189bf58 sp=0xc00189bea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc00189bfe0 sp=0xc00189bf58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00189bfe8 sp=0xc00189bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1073 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 97 [select]: +runtime.gopark(0xc000f6ff70?, 0x2?, 0xb4?, 0x1b?, 0xc000f6febc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f6fd20 sp=0xc000f6fd00 pc=0x4777ae +runtime.selectgo(0xc000f6ff70, 0xc000f6feb8, 0x0?, 0x0, 0x472ddb?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f6fe70 sp=0xc000f6fd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000996be0, {0x5b426e0, 0xc000762500}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc000f6ffa0 sp=0xc000f6fe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc000f6ffe0 sp=0xc000f6ffa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f6ffe8 sp=0xc000f6ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 162 [select]: +runtime.gopark(0xc000d13f70?, 0x2?, 0xb4?, 0x1b?, 0xc000d13ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010ebd20 sp=0xc0010ebd00 pc=0x4777ae +runtime.selectgo(0xc0010ebf70, 0xc000d13eb8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010ebe70 sp=0xc0010ebd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000996c40, {0x5b426e0, 0xc000762550}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0010ebfa0 sp=0xc0010ebe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0010ebfe0 sp=0xc0010ebfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010ebfe8 sp=0xc0010ebfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 159 [select]: +runtime.gopark(0xc000f6bda8?, 0x2?, 0x60?, 0xb8?, 0xc000f6ba94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f6b818 sp=0xc000f6b7f8 pc=0x4777ae +runtime.selectgo(0xc000f6bda8, 0xc000f6ba90, 0x0?, 0x0, 0x1860a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f6b968 sp=0xc000f6b818 pc=0x48916b +github.com/hashicorp/go-plugin.Serve(0xc000c4efc0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:474 +0x2091 fp=0xc000f6bfc0 sp=0xc000f6b968 pc=0x21665b1 +github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins.func4() + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x34 fp=0xc000f6bfe0 sp=0xc000f6bfc0 pc=0x24cd234 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f6bfe8 sp=0xc000f6bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins in goroutine 65 + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x76b + +goroutine 168 [select]: +runtime.gopark(0xc000a4bfb0?, 0x2?, 0xb4?, 0x1b?, 0xc000a4bf74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a4bde8 sp=0xc000a4bdc8 pc=0x4777ae +runtime.selectgo(0xc000a4bfb0, 0xc000a4bf70, 0xc000a4bfd0?, 0x0, 0xc0004bbd50?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a4bf38 sp=0xc000a4bde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc000a4bfe0 sp=0xc000a4bf38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a4bfe8 sp=0xc000a4bfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 160 [select]: +runtime.gopark(0xc000d08f48?, 0x2?, 0x20?, 0x8f?, 0xc000d08f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d08d80 sp=0xc000d08d60 pc=0x4777ae +runtime.selectgo(0xc000d08f48, 0xc000d08f00, 0x0?, 0x0, 0x20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d08ed0 sp=0xc000d08d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).Recv(0xc000c70510) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:124 +0xeb fp=0xc000d08f78 sp=0xc000d08ed0 pc=0x2154feb +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc000b763c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc000d08fc0 sp=0xc000d08f78 pc=0x215759f +github.com/hashicorp/go-plugin.(*GRPCServer).Init.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x34 fp=0xc000d08fe0 sp=0xc000d08fc0 pc=0x215adf4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d08fe8 sp=0xc000d08fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*GRPCServer).Init in goroutine 159 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x89d + +goroutine 487 [IO wait]: +runtime.gopark(0xc00157af88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00157af40 sp=0xc00157af20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00157af78 sp=0xc00157af40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6ea08, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00157af98 sp=0xc00157af78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014d3820, 0xc0014a0d80?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00157afe0 sp=0xc00157af98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0014d3800, {0xc0014a0d80, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00157b0d0 sp=0xc00157afe0 pc=0x50e5a5 +net.(*netFD).Read(0xc0014d3800, {0xc0014a0d80, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00157b130 sp=0xc00157b0d0 pc=0x89342b +net.(*conn).Read(0xc000640018, {0xc0014a0d80, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00157b1b8 sp=0xc00157b130 pc=0x8b176d +net.(*UnixConn).Read(0xc000640018, {0xc0014a0d80, 0x240, 0x240}) + :1 +0x54 fp=0xc00157b200 sp=0xc00157b1b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0004a7ba8, {0xc0014a0d80, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc00157b260 sp=0xc00157b200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc0d28, {0x5b0f6c0, 0xc0004a7ba8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc00157b2c8 sp=0xc00157b260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc0a80, {0x5b0f840?, 0xc000640018}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc00157b338 sp=0xc00157b2c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc0a80, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc00157b968 sp=0xc00157b338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc0a80, {0xc001776000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc00157ba50 sp=0xc00157b968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0016351e0, {0xc001776000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc00157ba98 sp=0xc00157ba50 pc=0x10969ed +bufio.(*Reader).Read(0xc0011a45a0, {0xc000e4ef20, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00157bb58 sp=0xc00157ba98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0011a45a0}, {0xc000e4ef20, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00157bba0 sp=0xc00157bb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000e4ef20, 0x9, 0x9}, {0x5b0f0a0, 0xc0011a45a0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00157bc40 sp=0xc00157bba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000e4eee0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00157bd78 sp=0xc00157bc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0012c5a00, 0x1000?, 0x5b0ee40?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc00157bec8 sp=0xc00157bd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00141a3c0, {0x5b54e00?, 0xc0012c5a00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc00157bf80 sp=0xc00157bec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc00157bfe0 sp=0xc00157bf80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00157bfe8 sp=0xc00157bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 453 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 486 [select]: +runtime.gopark(0xc000d3af70?, 0x4?, 0xa9?, 0x5?, 0xc000d3add8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d3ac00 sp=0xc000d3abe0 pc=0x4777ae +runtime.selectgo(0xc000d3af70, 0xc000d3add0, 0x87bab8?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d3ad50 sp=0xc000d3ac00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0012c5a00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc000d3afc0 sp=0xc000d3ad50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc000d3afe0 sp=0xc000d3afc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d3afe8 sp=0xc000d3afe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 453 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 101 [select]: +runtime.gopark(0xc000d6f748?, 0x2?, 0xd0?, 0xf5?, 0xc000d6f6fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d6f578 sp=0xc000d6f558 pc=0x4777ae +runtime.selectgo(0xc000d6f748, 0xc000d6f6f8, 0xc000934940?, 0x0, 0xc000d16198?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d6f6c8 sp=0xc000d6f578 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000926120) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc000d6f778 sp=0xc000d6f6c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00094c680, {0x43dd860?, 0xc000ee4dc0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc000d6f9d8 sp=0xc000d6f778 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000c05974?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc000d6fa38 sp=0xc000d6f9d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc000c058c0, 0xc000d6fbf0, 0xc000d6fbc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc000d6fae0 sp=0xc000d6fa38 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c058c0, {0x43dd860, 0xc000ee4dc0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc000d6fc20 sp=0xc000d6fae0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc000fe6ec0, {0x43dd860, 0xc000ee4dc0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc000d6fde8 sp=0xc000d6fc20 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamClient).Recv(0xc000b6f0f0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:132 +0x66 fp=0xc000d6fe28 sp=0xc000d6fde8 pc=0x2102606 +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream(0xc000fe6b80) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:197 +0x2e8 fp=0xc000d6ffc0 sp=0xc000d6fe28 pc=0x2155468 +github.com/hashicorp/go-plugin.newGRPCClient.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x34 fp=0xc000d6ffe0 sp=0xc000d6ffc0 pc=0x2159954 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d6ffe8 sp=0xc000d6ffe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 154 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x62e + +goroutine 179 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e81940 sp=0xc000e81920 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000e81978 sp=0xc000e81940 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6e720, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000e81998 sp=0xc000e81978 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000c43ea0, 0xc000e81a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000e819e0 sp=0xc000e81998 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000c43e80) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc000e81ad0 sp=0xc000e819e0 pc=0x513f65 +net.(*netFD).accept(0xc000c43e80) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc000e81bc8 sp=0xc000e81ad0 pc=0x8967be +net.(*UnixListener).accept(0xc000c70150) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc000e81c08 sp=0xc000e81bc8 pc=0x8cedfe +net.(*UnixListener).Accept(0xc000c70150) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc000e81c78 sp=0xc000e81c08 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc000b75280) + :1 +0x44 fp=0xc000e81cb0 sp=0xc000e81c78 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc000c40d20, {0x5b2c6a0, 0xc000b75280}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc000e81ee0 sp=0xc000e81cb0 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCServer).Serve(0xc000bbcf30, {0x5b2c6a0, 0xc000b75280}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:153 +0xdd fp=0xc000e81f98 sp=0xc000e81ee0 pc=0x215b33d +github.com/hashicorp/go-plugin.Serve.func6() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x62 fp=0xc000e81fe0 sp=0xc000e81f98 pc=0x2166b62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e81fe8 sp=0xc000e81fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.Serve in goroutine 159 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x1f67 + +goroutine 169 [select]: +runtime.gopark(0xc000d04fb0?, 0x2?, 0xb0?, 0x4d?, 0xc000d04ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d04d58 sp=0xc000d04d38 pc=0x4777ae +runtime.selectgo(0xc000d04fb0, 0xc000d04ee0, 0xc000d04f98?, 0x0, 0xc000dddfb0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d04ea8 sp=0xc000d04d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc000d04fe0 sp=0xc000d04ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d04fe8 sp=0xc000d04fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 166 [IO wait]: +runtime.gopark(0xc000e90fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e90f98 sp=0xc000e90f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000e90fd0 sp=0xc000e90f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6e248, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000e90ff0 sp=0xc000e90fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d72620, 0xc001749800?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000e91038 sp=0xc000e90ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000d72600, {0xc001749800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000e91128 sp=0xc000e91038 pc=0x50e5a5 +net.(*netFD).Read(0xc000d72600, {0xc001749800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc000e91188 sp=0xc000e91128 pc=0x89342b +net.(*conn).Read(0xc00007d870, {0xc001749800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc000e91210 sp=0xc000e91188 pc=0x8b176d +net.(*UnixConn).Read(0xc00007d870, {0xc001749800, 0x4800, 0x4800}) + :1 +0x54 fp=0xc000e91258 sp=0xc000e91210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0008630f8, {0xc001749800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc000e912b8 sp=0xc000e91258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000c7e9a8, {0x5b0f6c0, 0xc0008630f8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc000e91320 sp=0xc000e912b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000c7e700, {0x5b0f840?, 0xc00007d870}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc000e91390 sp=0xc000e91320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000c7e700, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc000e919c0 sp=0xc000e91390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000c7e700, {0xc000fb8000, 0x8000, 0xc000bdc050?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc000e91aa8 sp=0xc000e919c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0009970e0, {0xc000fb8000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc000e91af0 sp=0xc000e91aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc000f1b560, {0xc000db2820, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc000e91bb0 sp=0xc000e91af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000f1b560}, {0xc000db2820, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000e91bf8 sp=0xc000e91bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000db2820, 0x9, 0x9}, {0x5b0f0a0, 0xc000f1b560}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc000e91c98 sp=0xc000e91bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000db27e0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc000e91dd0 sp=0xc000e91c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000eecfc0, 0xc000c7e700?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc000e91fb0 sp=0xc000e91dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc000e91fe0 sp=0xc000e91fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e91fe8 sp=0xc000e91fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 163 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 185 [select]: +runtime.gopark(0xc002a7fe48?, 0x2?, 0x0?, 0x0?, 0xc002a7fdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002a7fc30 sp=0xc002a7fc10 pc=0x4777ae +runtime.selectgo(0xc002a7fe48, 0xc002a7fdc8, 0xc002a7fdd8?, 0x0, 0xc000dc0000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002a7fd80 sp=0xc002a7fc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000bdda90, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002a7fe78 sp=0xc002a7fd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004aad20) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002a7ff30 sp=0xc002a7fe78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002a7ffe0 sp=0xc002a7ff30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002a7ffe8 sp=0xc002a7ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 181 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 135 [chan receive]: +runtime.gopark(0x0?, 0x10?, 0xa0?, 0xf0?, 0xc002a4b5d4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d3bd10 sp=0xc000d3bcf0 pc=0x4777ae +runtime.chanrecv(0xc000c4f320, 0xc000d3be40, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000d3bd88 sp=0xc000d3bd10 pc=0x441325 +runtime.chanrecv2(0x3b9aca00?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc000d3bdb0 sp=0xc000d3bd88 pc=0x440f92 +github.com/hashicorp/go-plugin.pidWait(0x1860a0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/process.go:20 +0xdb fp=0xc000d3be68 sp=0xc000d3bdb0 pc=0x216033b +github.com/hashicorp/go-plugin.(*Client).reattach.func1(0x215adf4?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:853 +0xd8 fp=0xc000d3bfb8 sp=0xc000d3be68 pc=0x21528b8 +github.com/hashicorp/go-plugin.(*Client).reattach.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:862 +0x42 fp=0xc000d3bfe0 sp=0xc000d3bfb8 pc=0x21527a2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d3bfe8 sp=0xc000d3bfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*Client).reattach in goroutine 134 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:846 +0x365 + +goroutine 142 [select]: +runtime.gopark(0xc000d36fb0?, 0x2?, 0xb0?, 0x6d?, 0xc000d36ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d36d58 sp=0xc000d36d38 pc=0x4777ae +runtime.selectgo(0xc000d36fb0, 0xc000d36ee0, 0xc000d36f98?, 0x0, 0xc000d183c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d36ea8 sp=0xc000d36d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc000d36fe0 sp=0xc000d36ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d36fe8 sp=0xc000d36fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 136 [select]: +runtime.gopark(0xc000e95f70?, 0x2?, 0xb4?, 0x1b?, 0xc000e95ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e95d20 sp=0xc000e95d00 pc=0x4777ae +runtime.selectgo(0xc000e95f70, 0xc000e95eb8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000e95e70 sp=0xc000e95d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0005fad80, {0x5b426e0, 0xc000bdcb40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc000e95fa0 sp=0xc000e95e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc000e95fe0 sp=0xc000e95fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e95fe8 sp=0xc000e95fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 137 [select]: +runtime.gopark(0xc000a71f70?, 0x2?, 0xb4?, 0x1b?, 0xc000a71ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a71d20 sp=0xc000a71d00 pc=0x4777ae +runtime.selectgo(0xc000a71f70, 0xc000a71eb8, 0x0?, 0x0, 0x440f72?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a71e70 sp=0xc000a71d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0005fade0, {0x5b426e0, 0xc000bdcb90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc000a71fa0 sp=0xc000a71e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc000a71fe0 sp=0xc000a71fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a71fe8 sp=0xc000a71fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 141 [select]: +runtime.gopark(0xc000d37fb0?, 0x2?, 0xb4?, 0x1b?, 0xc000d37f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d37de8 sp=0xc000d37dc8 pc=0x4777ae +runtime.selectgo(0xc000d37fb0, 0xc000d37f70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d37f38 sp=0xc000d37de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc000d37fe0 sp=0xc000d37f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d37fe8 sp=0xc000d37fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 139 [select]: +runtime.gopark(0xc000d09f48?, 0x2?, 0x0?, 0xda?, 0xc000d09f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d09d80 sp=0xc000d09d60 pc=0x4777ae +runtime.selectgo(0xc000d09f48, 0xc000d09f00, 0xc000d09f08?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d09ed0 sp=0xc000d09d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).Recv(0xc0008c4440) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:234 +0xeb fp=0xc000d09f78 sp=0xc000d09ed0 pc=0x2155d2b +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc000bdd540) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc000d09fc0 sp=0xc000d09f78 pc=0x215759f +github.com/hashicorp/go-plugin.newGRPCClient.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x34 fp=0xc000d09fe0 sp=0xc000d09fc0 pc=0x21599b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d09fe8 sp=0xc000d09fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 134 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x5b7 + +goroutine 140 [select]: +runtime.gopark(0xc000d6b748?, 0x2?, 0xd0?, 0xb5?, 0xc000d6b6fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d6b578 sp=0xc000d6b558 pc=0x4777ae +runtime.selectgo(0xc000d6b748, 0xc000d6b6f8, 0xc000a52580?, 0x0, 0xc000ed2e18?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d6b6c8 sp=0xc000d6b578 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000926360) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc000d6b778 sp=0xc000d6b6c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00089fd40, {0x43dd860?, 0xc000ee4f00}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc000d6b9d8 sp=0xc000d6b778 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00094e894?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc000d6ba38 sp=0xc000d6b9d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00094e7e0, 0xc000d6bbf0, 0xc000d6bbc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc000d6bae0 sp=0xc000d6ba38 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00094e7e0, {0x43dd860, 0xc000ee4f00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc000d6bc20 sp=0xc000d6bae0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc000fe6fc0, {0x43dd860, 0xc000ee4f00}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc000d6bde8 sp=0xc000d6bc20 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamClient).Recv(0xc000b6f1c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:132 +0x66 fp=0xc000d6be28 sp=0xc000d6bde8 pc=0x2102606 +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream(0xc0008c4440) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:197 +0x2e8 fp=0xc000d6bfc0 sp=0xc000d6be28 pc=0x2155468 +github.com/hashicorp/go-plugin.newGRPCClient.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x34 fp=0xc000d6bfe0 sp=0xc000d6bfc0 pc=0x2159954 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d6bfe8 sp=0xc000d6bfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 134 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x62e + +goroutine 105 [IO wait]: +runtime.gopark(0xc000e92fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e92f98 sp=0xc000e92f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000e92fd0 sp=0xc000e92f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6e058, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000e92ff0 sp=0xc000e92fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d163a0, 0xc001a0e000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000e93038 sp=0xc000e92ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000d16380, {0xc001a0e000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000e93128 sp=0xc000e93038 pc=0x50e5a5 +net.(*netFD).Read(0xc000d16380, {0xc001a0e000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc000e93188 sp=0xc000e93128 pc=0x89342b +net.(*conn).Read(0xc000991078, {0xc001a0e000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc000e93210 sp=0xc000e93188 pc=0x8b176d +net.(*UnixConn).Read(0xc000991078, {0xc001a0e000, 0xa000, 0xa000}) + :1 +0x54 fp=0xc000e93258 sp=0xc000e93210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00290e0d8, {0xc001a0e000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc000e932b8 sp=0xc000e93258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000f889a8, {0x5b0f6c0, 0xc00290e0d8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc000e93320 sp=0xc000e932b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000f88700, {0x5b0f840?, 0xc000991078}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc000e93390 sp=0xc000e93320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000f88700, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc000e939c0 sp=0xc000e93390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000f88700, {0xc000fd8000, 0x8000, 0xc00296def0?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc000e93aa8 sp=0xc000e939c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc000ee6820, {0xc000fd8000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc000e93af0 sp=0xc000e93aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc000d65a40, {0xc000f8e040, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc000e93bb0 sp=0xc000e93af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000d65a40}, {0xc000f8e040, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000e93bf8 sp=0xc000e93bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000f8e040, 0x9, 0x9}, {0x5b0f0a0, 0xc000d65a40}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc000e93c98 sp=0xc000e93bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000f8e000) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc000e93dd0 sp=0xc000e93c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000b50fc0, 0xc000f88700?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc000e93fb0 sp=0xc000e93dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc000e93fe0 sp=0xc000e93fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e93fe8 sp=0xc000e93fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 138 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 188 [select]: +runtime.gopark(0xc000e7d310?, 0x3?, 0xd9?, 0xea?, 0xc000e7d292?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e7d0f0 sp=0xc000e7d0d0 pc=0x4777ae +runtime.selectgo(0xc000e7d310, 0xc000e7d28c, 0x46216a2?, 0x0, 0x4?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000e7d240 sp=0xc000e7d0f0 pc=0x48916b +github.com/hashicorp/go-plugin.(*grpcStdioServer).StreamStdio(0xc000bfd4e0, 0x42219c0?, {0x5b4ebb0, 0xc000b6f570}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:61 +0x1e5 fp=0xc000e7d350 sp=0xc000e7d240 pc=0x215bbc5 +github.com/hashicorp/go-plugin/internal/plugin._GRPCStdio_StreamStdio_Handler({0x409e580?, 0xc000bfd4e0}, {0x5b4a6f0, 0xc000ee69c0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:185 +0xf7 fp=0xc000e7d3b0 sp=0xc000e7d350 pc=0x2104077 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x409e580, 0xc000bfd4e0}, {0x5b4ab30, 0xc000d19170}, 0xc000f1ec78?, 0x47efca8) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000e7d400 sp=0xc000e7d3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x409e580, 0xc000bfd4e0}, {0x5b4ab30, 0xc000d19170}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000e7d490 sp=0xc000e7d400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x409e580, 0xc000bfd4e0}, {0x5b492f8, 0xc0009ec780}, 0xc000f1ec78, 0xc000fe7040) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000e7d7d0 sp=0xc000e7d490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x409e580, 0xc000bfd4e0}, {0x5b492f8, 0xc0009ec780}, 0x3f252a0?, 0xc000b6f1d0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000e7d848 sp=0xc000e7d7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc000e51320, 0xc000c4a870, 0x770cba0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000e7ddb0 sp=0xc000e7d848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc000e51320, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000e7df28 sp=0xc000e7ddb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000e7dfe0 sp=0xc000e7df28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e7dfe8 sp=0xc000e7dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 167 [select]: +runtime.gopark(0xc000c59e70?, 0x2?, 0x0?, 0x0?, 0xc000c59df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020dfc58 sp=0xc0020dfc38 pc=0x4777ae +runtime.selectgo(0xc0020dfe70, 0xc000c59df0, 0xc000c59e00?, 0x0, 0xc000c7e700?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020dfda8 sp=0xc0020dfc58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000763360, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0020dfea0 sp=0xc0020dfda8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004bbd50) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0020dff58 sp=0xc0020dfea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0020dffe0 sp=0xc0020dff58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020dffe8 sp=0xc0020dffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 163 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 182 [select]: +runtime.gopark(0xc001f3fe48?, 0x2?, 0x0?, 0x0?, 0xc001f3fdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e69c30 sp=0xc001e69c10 pc=0x4777ae +runtime.selectgo(0xc001e69e48, 0xc001f3fdc8, 0xc001f3fdd8?, 0x0, 0xc000f88000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e69d80 sp=0xc001e69c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000b76550, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001e69e78 sp=0xc001e69d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004aac40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001e69f30 sp=0xc001e69e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc001e69fe0 sp=0xc001e69f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e69fe8 sp=0xc001e69fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 180 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 183 [select]: +runtime.gopark(0xc000d3cf70?, 0x4?, 0x10?, 0x0?, 0xc000d3cdd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d3cc00 sp=0xc000d3cbe0 pc=0x4777ae +runtime.selectgo(0xc000d3cf70, 0xc000d3cdd0, 0xc000c48fe0?, 0x0, 0x112e2bb?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d3cd50 sp=0xc000d3cc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000c7cb60) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc000d3cfc0 sp=0xc000d3cd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc000d3cfe0 sp=0xc000d3cfc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d3cfe8 sp=0xc000d3cfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 180 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 184 [IO wait]: +runtime.gopark(0xc000e82f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e82f40 sp=0xc000e82f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000e82f78 sp=0xc000e82f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6e150, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000e82f98 sp=0xc000e82f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000dba0a0, 0xc00024c800?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000e82fe0 sp=0xc000e82f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000dba080, {0xc00024c800, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000e830d0 sp=0xc000e82fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc000dba080, {0xc00024c800, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc000e83130 sp=0xc000e830d0 pc=0x89342b +net.(*conn).Read(0xc000b799b8, {0xc00024c800, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc000e831b8 sp=0xc000e83130 pc=0x8b176d +net.(*UnixConn).Read(0xc000b799b8, {0xc00024c800, 0x800, 0x800}) + :1 +0x54 fp=0xc000e83200 sp=0xc000e831b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc000013f80, {0xc00024c800, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc000e83260 sp=0xc000e83200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000f882a8, {0x5b0f6c0, 0xc000013f80}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc000e832c8 sp=0xc000e83260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000f88000, {0x5b0f840?, 0xc000b799b8}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc000e83338 sp=0xc000e832c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000f88000, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc000e83968 sp=0xc000e83338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000f88000, {0xc000fd0000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc000e83a50 sp=0xc000e83968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc000b756c0, {0xc000fd0000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc000e83a98 sp=0xc000e83a50 pc=0x10969ed +bufio.(*Reader).Read(0xc000c4f680, {0xc000c49000, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc000e83b58 sp=0xc000e83a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000c4f680}, {0xc000c49000, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000e83ba0 sp=0xc000e83b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000c49000, 0x9, 0x9}, {0x5b0f0a0, 0xc000c4f680}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc000e83c40 sp=0xc000e83ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000c48fc0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc000e83d78 sp=0xc000e83c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000c7cb60, 0x0?, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc000e83ec8 sp=0xc000e83d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc000c401e0, {0x5b54e00?, 0xc000c7cb60}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc000e83f80 sp=0xc000e83ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc000e83fe0 sp=0xc000e83f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e83fe8 sp=0xc000e83fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 180 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 106 [select]: +runtime.gopark(0xc00111fe70?, 0x2?, 0x0?, 0x0?, 0xc00111fdf4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00111fc58 sp=0xc00111fc38 pc=0x4777ae +runtime.selectgo(0xc00111fe70, 0xc00111fdf0, 0xc00111fe00?, 0x0, 0xc000f88700?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00111fda8 sp=0xc00111fc58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000ee4c30, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00111fea0 sp=0xc00111fda8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004b4af0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00111ff58 sp=0xc00111fea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc00111ffe0 sp=0xc00111ff58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00111ffe8 sp=0xc00111ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 138 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 170 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc000ef8df8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000ef8d60 sp=0xc000ef8d40 pc=0x4777ae +runtime.chanrecv(0xc000f26060, 0xc000ef8e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000ef8dd8 sp=0xc000ef8d60 pc=0x441325 +runtime.chanrecv1(0x448845?, 0x40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc000ef8e00 sp=0xc000ef8dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc000ef8fe0 sp=0xc000ef8e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000ef8fe8 sp=0xc000ef8fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 171 [select]: +runtime.gopark(0xc000e97790?, 0x2?, 0x18?, 0x76?, 0xc000e97744?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e975c0 sp=0xc000e975a0 pc=0x4777ae +runtime.selectgo(0xc000e97790, 0xc000e97740, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000e97710 sp=0xc000e975c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000f8aa20) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc000e977c0 sp=0xc000e97710 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00094c4e0, {0x43c4ea0?, 0xc000fe7000}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc000e97a20 sp=0xc000e977c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000c05854?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc000e97a80 sp=0xc000e97a20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc000c057a0, 0xc000ef8c38, 0xc000ef8c08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc000e97b28 sp=0xc000e97a80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c057a0, {0x43c4ea0, 0xc000fe7000}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc000e97c68 sp=0xc000e97b28 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc000dd8e40, {0x43c4ea0, 0xc000fe7000}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc000e97e30 sp=0xc000e97c68 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCStdioStreamStdioClient).Recv(0xc000813b10) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:160 +0x66 fp=0xc000e97e70 sp=0xc000e97e30 pc=0x2103f26 +github.com/hashicorp/go-plugin.(*grpcStdioClient).Run(0xc000997200, {0x5b0ee60, 0x779c980}, {0x5b0eb80, 0xc00007c038}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:135 +0x11c fp=0xc000e97f80 sp=0xc000e97e70 pc=0x215c15c +github.com/hashicorp/go-plugin.newGRPCClient.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x6c fp=0xc000e97fe0 sp=0xc000e97f80 pc=0x21598ec +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e97fe8 sp=0xc000e97fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 154 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x84f + +goroutine 186 [select]: +runtime.gopark(0xc000d3df70?, 0x4?, 0x58?, 0xda?, 0xc000d3ddd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d3dc00 sp=0xc000d3dbe0 pc=0x4777ae +runtime.selectgo(0xc000d3df70, 0xc000d3ddd0, 0x40a16a0?, 0x0, 0xc000e9c050?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d3dd50 sp=0xc000d3dc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000e8c680) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc000d3dfc0 sp=0xc000d3dd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc000d3dfe0 sp=0xc000d3dfc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d3dfe8 sp=0xc000d3dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 181 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 187 [IO wait]: +runtime.gopark(0xc00109ef88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00109ef40 sp=0xc00109ef20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00109ef78 sp=0xc00109ef40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677eb0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00109ef98 sp=0xc00109ef78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000dba220, 0xc001154000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00109efe0 sp=0xc00109ef98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000dba200, {0xc001154000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00109f0d0 sp=0xc00109efe0 pc=0x50e5a5 +net.(*netFD).Read(0xc000dba200, {0xc001154000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00109f130 sp=0xc00109f0d0 pc=0x89342b +net.(*conn).Read(0xc000b799c0, {0xc001154000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00109f1b8 sp=0xc00109f130 pc=0x8b176d +net.(*UnixConn).Read(0xc000b799c0, {0xc001154000, 0x800, 0x800}) + :1 +0x54 fp=0xc00109f200 sp=0xc00109f1b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0027d83c0, {0xc001154000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc00109f260 sp=0xc00109f200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc02a8, {0x5b0f6c0, 0xc0027d83c0}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc00109f2c8 sp=0xc00109f260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc0000, {0x5b0f840?, 0xc000b799c0}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc00109f338 sp=0xc00109f2c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc0000, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc00109f968 sp=0xc00109f338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc0000, {0xc000e9e000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc00109fa50 sp=0xc00109f968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0005fafe0, {0xc000e9e000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc00109fa98 sp=0xc00109fa50 pc=0x10969ed +bufio.(*Reader).Read(0xc000a528a0, {0xc000477380, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00109fb58 sp=0xc00109fa98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000a528a0}, {0xc000477380, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00109fba0 sp=0xc00109fb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000477380, 0x9, 0x9}, {0x5b0f0a0, 0xc000a528a0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00109fc40 sp=0xc00109fba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000477340) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00109fd78 sp=0xc00109fc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000e8c680, 0x1177efa?, 0x215b33d?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc00109fec8 sp=0xc00109fd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc000c40d20, {0x5b54e00?, 0xc000e8c680}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc00109ff80 sp=0xc00109fec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc00109ffe0 sp=0xc00109ff80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00109ffe8 sp=0xc00109ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 181 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 143 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x1188140?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d05d60 sp=0xc000d05d40 pc=0x4777ae +runtime.chanrecv(0xc000c9d2c0, 0xc000d05e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000d05dd8 sp=0xc000d05d60 pc=0x441325 +runtime.chanrecv1(0x472ddb?, 0xc000bdd7c0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc000d05e00 sp=0xc000d05dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc000d05fe0 sp=0xc000d05e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d05fe8 sp=0xc000d05fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 172 [select]: +runtime.gopark(0xc000a4dfb0?, 0x2?, 0xb4?, 0x1b?, 0xc000a4df74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a4dde8 sp=0xc000a4ddc8 pc=0x4777ae +runtime.selectgo(0xc000a4dfb0, 0xc000a4df70, 0x2?, 0x0, 0x5b42750?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a4df38 sp=0xc000a4dde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc000a4dfe0 sp=0xc000a4df38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a4dfe8 sp=0xc000a4dfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 173 [select]: +runtime.gopark(0xc000a4cfb0?, 0x2?, 0xb0?, 0xcd?, 0xc000a4cee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a4cd58 sp=0xc000a4cd38 pc=0x4777ae +runtime.selectgo(0xc000a4cfb0, 0xc000a4cee0, 0x448855?, 0x0, 0x400a060?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a4cea8 sp=0xc000a4cd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc000a4cfe0 sp=0xc000a4cea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a4cfe8 sp=0xc000a4cfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 107 [select]: +runtime.gopark(0xc000d39fb0?, 0x2?, 0xb4?, 0x1b?, 0xc000d39f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d39de8 sp=0xc000d39dc8 pc=0x4777ae +runtime.selectgo(0xc000d39fb0, 0xc000d39f70, 0xc000b85fd0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d39f38 sp=0xc000d39de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc000d39fe0 sp=0xc000d39f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d39fe8 sp=0xc000d39fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 101 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 174 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000ef9d60 sp=0xc000ef9d40 pc=0x4777ae +runtime.chanrecv(0xc000f26300, 0xc000ef9e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000ef9dd8 sp=0xc000ef9d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc000ef9e00 sp=0xc000ef9dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc000ef9fe0 sp=0xc000ef9e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000ef9fe8 sp=0xc000ef9fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 108 [select]: +runtime.gopark(0xc001092fb0?, 0x2?, 0xb0?, 0x2d?, 0xc001092ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001092d58 sp=0xc001092d38 pc=0x4777ae +runtime.selectgo(0xc001092fb0, 0xc001092ee0, 0x392effa?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001092ea8 sp=0xc001092d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001092fe0 sp=0xc001092ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001092fe8 sp=0xc001092fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 101 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 109 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001093d60 sp=0xc001093d40 pc=0x4777ae +runtime.chanrecv(0xc001089140, 0xc001093e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001093dd8 sp=0xc001093d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001093e00 sp=0xc001093dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001093fe0 sp=0xc001093e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001093fe8 sp=0xc001093fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 101 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 144 [select]: +runtime.gopark(0xc00109b790?, 0x2?, 0x18?, 0xb6?, 0xc00109b744?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00109b5c0 sp=0xc00109b5a0 pc=0x4777ae +runtime.selectgo(0xc00109b790, 0xc00109b740, 0x1?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00109b710 sp=0xc00109b5c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00094eea0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc00109b7c0 sp=0xc00109b710 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00089fc70, {0x43c4ea0?, 0xc0008c4880}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc00109ba20 sp=0xc00109b7c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00094e774?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc00109ba80 sp=0xc00109ba20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00094e6c0, 0xc000efac38, 0xc000efac08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc00109bb28 sp=0xc00109ba80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00094e6c0, {0x43c4ea0, 0xc0008c4880}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc00109bc68 sp=0xc00109bb28 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0008c4600, {0x43c4ea0, 0xc0008c4880}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc00109be30 sp=0xc00109bc68 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCStdioStreamStdioClient).Recv(0xc000e9c1c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:160 +0x66 fp=0xc00109be70 sp=0xc00109be30 pc=0x2103f26 +github.com/hashicorp/go-plugin.(*grpcStdioClient).Run(0xc0005fb100, {0x5b0ee60, 0x779c980}, {0x5b0eb80, 0xc00007c038}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:135 +0x11c fp=0xc00109bf80 sp=0xc00109be70 pc=0x215c15c +github.com/hashicorp/go-plugin.newGRPCClient.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x6c fp=0xc00109bfe0 sp=0xc00109bf80 pc=0x21598ec +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00109bfe8 sp=0xc00109bfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 134 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x84f + +goroutine 110 [select]: +runtime.gopark(0xc0015f3fa0?, 0x3?, 0xf0?, 0x3d?, 0xc0015f3f3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015f3da8 sp=0xc0015f3d88 pc=0x4777ae +runtime.selectgo(0xc0015f3fa0, 0xc0015f3f34, 0xc000d38f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0015f3ef8 sp=0xc0015f3da8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:184 +0x1b3 fp=0xc0015f3fe0 sp=0xc0015f3ef8 pc=0x21557f3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015f3fe8 sp=0xc0015f3fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream in goroutine 101 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:182 +0x24a + +goroutine 145 [select]: +runtime.gopark(0xc000a4afb0?, 0x2?, 0xb4?, 0x1b?, 0xc000a4af74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a4ade8 sp=0xc000a4adc8 pc=0x4777ae +runtime.selectgo(0xc000a4afb0, 0xc000a4af70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a4af38 sp=0xc000a4ade8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc000a4afe0 sp=0xc000a4af38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a4afe8 sp=0xc000a4afe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 194 [select]: +runtime.gopark(0xc000efafb0?, 0x2?, 0xb0?, 0xad?, 0xc000efaee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000efad58 sp=0xc000efad38 pc=0x4777ae +runtime.selectgo(0xc000efafb0, 0xc000efaee0, 0x4aead9?, 0x0, 0xc000f1df30?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000efaea8 sp=0xc000efad58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc000efafe0 sp=0xc000efaea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000efafe8 sp=0xc000efafe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 195 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000efbd60 sp=0xc000efbd40 pc=0x4777ae +runtime.chanrecv(0xc000c9d560, 0xc000efbe60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000efbdd8 sp=0xc000efbd60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc000efbe00 sp=0xc000efbdd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc000efbfe0 sp=0xc000efbe00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000efbfe8 sp=0xc000efbfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 111 [select]: +runtime.gopark(0xc001095fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001095f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001095de8 sp=0xc001095dc8 pc=0x4777ae +runtime.selectgo(0xc001095fb0, 0xc001095f70, 0x1065963d?, 0x0, 0xc000c70150?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001095f38 sp=0xc001095de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001095fe0 sp=0xc001095f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001095fe8 sp=0xc001095fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 140 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 112 [select]: +runtime.gopark(0xc001094fb0?, 0x2?, 0xb0?, 0x4d?, 0xc001094ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001094d58 sp=0xc001094d38 pc=0x4777ae +runtime.selectgo(0xc001094fb0, 0xc001094ee0, 0x392effa?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001094ea8 sp=0xc001094d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001094fe0 sp=0xc001094ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001094fe8 sp=0xc001094fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 140 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 113 [chan receive]: +runtime.gopark(0xc000ef6e30?, 0x1?, 0x1?, 0x0?, 0x3?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000ef6d60 sp=0xc000ef6d40 pc=0x4777ae +runtime.chanrecv(0xc001089380, 0xc000ef6e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000ef6dd8 sp=0xc000ef6d60 pc=0x441325 +runtime.chanrecv1(0xc00055e1e8?, 0xc000553570?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc000ef6e00 sp=0xc000ef6dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc000ef6fe0 sp=0xc000ef6e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000ef6fe8 sp=0xc000ef6fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 140 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 210 [select]: +runtime.gopark(0xc001131fa0?, 0x3?, 0xf0?, 0x1d?, 0xc001131f3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001495da8 sp=0xc001495d88 pc=0x4777ae +runtime.selectgo(0xc001495fa0, 0xc001131f34, 0xc000ef7f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001495ef8 sp=0xc001495da8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:184 +0x1b3 fp=0xc001495fe0 sp=0xc001495ef8 pc=0x21557f3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001495fe8 sp=0xc001495fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream in goroutine 140 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:182 +0x24a + +goroutine 196 [select]: +runtime.gopark(0xc000e7f310?, 0x3?, 0xd9?, 0xea?, 0xc000e7f292?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000e7f0f0 sp=0xc000e7f0d0 pc=0x4777ae +runtime.selectgo(0xc000e7f310, 0xc000e7f28c, 0x46216a2?, 0x0, 0x4?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000e7f240 sp=0xc000e7f0f0 pc=0x48916b +github.com/hashicorp/go-plugin.(*grpcStdioServer).StreamStdio(0xc000c6e680, 0x42219c0?, {0x5b4ebb0, 0xc000b6f820}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:61 +0x1e5 fp=0xc000e7f350 sp=0xc000e7f240 pc=0x215bbc5 +github.com/hashicorp/go-plugin/internal/plugin._GRPCStdio_StreamStdio_Handler({0x409e580?, 0xc000c6e680}, {0x5b4a6f0, 0xc000ee6a20}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:185 +0xf7 fp=0xc000e7f3b0 sp=0xc000e7f350 pc=0x2104077 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x409e580, 0xc000c6e680}, {0x5b4ab30, 0xc000d193b0}, 0xc000f1ecc0?, 0x47efca8) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000e7f400 sp=0xc000e7f3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x409e580, 0xc000c6e680}, {0x5b4ab30, 0xc000d193b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000e7f490 sp=0xc000e7f400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x409e580, 0xc000c6e680}, {0x5b492f8, 0xc0009ecf00}, 0xc000f1ecc0, 0xc000fe7180) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000e7f7d0 sp=0xc000e7f490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x409e580, 0xc000c6e680}, {0x5b492f8, 0xc0009ecf00}, 0x3f252a0?, 0xc000b6f5b0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000e7f848 sp=0xc000e7f7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00094f320, 0xc000c70750, 0x770cba0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000e7fdb0 sp=0xc000e7f848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00094f320, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000e7ff28 sp=0xc000e7fdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000e7ffe0 sp=0xc000e7ff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000e7ffe8 sp=0xc000e7ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 189 [select]: +runtime.gopark(0xc000f0e890?, 0x2?, 0x4e?, 0x1a?, 0xc000f0e83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f0e6a8 sp=0xc000f0e688 pc=0x4777ae +runtime.selectgo(0xc000f0e890, 0xc000f0e838, 0x7f429a410b00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f0e7f8 sp=0xc000f0e6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000b76b90, {0xc000c71bd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc000f0e8c0 sp=0xc000f0e7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000b76b90, {0xc000c71bd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc000f0e958 sp=0xc000f0e8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000c719b0, {0xc000c71bd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc000f0e9a0 sp=0xc000f0e958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000c719b0}, {0xc000c71bd0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000f0e9e8 sp=0xc000f0e9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000e51440, {0xc000c71bd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc000f0ea58 sp=0xc000f0e9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000c71bc0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc000f0eb28 sp=0xc000f0ea58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0010b60c0?, 0xc000e51440, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc000f0ec18 sp=0xc000f0eb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000c71bc0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0009d60f0}, 0xc000c4a360?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc000f0ecd0 sp=0xc000f0ec18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000c071d0, {0x43dcf60?, 0xc0009d60f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc000f0ef08 sp=0xc000f0ecd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc000c71d10, {0x43dcf60, 0xc0009d60f0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc000f0eff0 sp=0xc000f0ef08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000b75860, {0x43dcf60, 0xc0009d60f0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc000f0f028 sp=0xc000f0eff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc000c6ede0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc000f0f068 sp=0xc000f0f028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc000c6ede0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc000f0f0a8 sp=0xc000f0f068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc000c6edf0) + :1 +0x46 fp=0xc000f0f0e0 sp=0xc000f0f0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c4a4e0, {0x5b51368, 0xc000c6edf0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc000f0f320 sp=0xc000f0f0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000bfd440, {0x5b51038, 0xc000c6ede0}) + :1 +0xaf fp=0xc000f0f370 sp=0xc000f0f320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000bfd440}, {0x5b4a6f0?, 0xc000b75860}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc000f0f3b0 sp=0xc000f0f370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc000c71d10}, 0xc000dc21b0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000f0f400 sp=0xc000f0f3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc000c71d10}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000f0f490 sp=0xc000f0f400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000c071d0}, 0xc000dc21b0, 0xc00091ca40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000f0f7d0 sp=0xc000f0f490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000c071d0}, 0x3f252a0?, 0xc000c6ed10?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000f0f848 sp=0xc000f0f7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc000e51440, 0xc000c4a570, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000f0fdb0 sp=0xc000f0f848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc000e51440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000f0ff28 sp=0xc000f0fdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000f0ffe0 sp=0xc000f0ff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f0ffe8 sp=0xc000f0ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 197 [select]: +runtime.gopark(0xc000d0e890?, 0x2?, 0x4e?, 0x1a?, 0xc000d0e83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d0e6a8 sp=0xc000d0e688 pc=0x4777ae +runtime.selectgo(0xc000d0e890, 0xc000d0e838, 0x7f429a410800?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d0e7f8 sp=0xc000d0e6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0000fc730, {0xc000e394b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc000d0e8c0 sp=0xc000d0e7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0000fc730, {0xc000e394b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc000d0e958 sp=0xc000d0e8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000eb8360, {0xc000e394b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc000d0e9a0 sp=0xc000d0e958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000eb8360}, {0xc000e394b0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000d0e9e8 sp=0xc000d0e9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00094f440, {0xc000e394b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc000d0ea58 sp=0xc000d0e9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000e394a0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc000d0eb28 sp=0xc000d0ea58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0010b6080?, 0xc00094f440, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc000d0ec18 sp=0xc000d0eb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000e394a0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0009d6000}, 0x13908f0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc000d0ecd0 sp=0xc000d0ec18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0010a62d0, {0x43dcf60?, 0xc0009d6000}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc000d0ef08 sp=0xc000d0ecd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc000e395f0, {0x43dcf60, 0xc0009d6000}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc000d0eff0 sp=0xc000d0ef08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0009974e0, {0x43dcf60, 0xc0009d6000}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc000d0f028 sp=0xc000d0eff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc000813fc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc000d0f068 sp=0xc000d0f028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc000813fc0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc000d0f0a8 sp=0xc000d0f068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc000813fd0) + :1 +0x46 fp=0xc000d0f0e0 sp=0xc000d0f0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c703c0, {0x5b51368, 0xc000813fd0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc000d0f320 sp=0xc000d0f0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000c6e5e0, {0x5b51038, 0xc000813fc0}) + :1 +0xaf fp=0xc000d0f370 sp=0xc000d0f320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000c6e5e0}, {0x5b4a6f0?, 0xc0009974e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc000d0f3b0 sp=0xc000d0f370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc000e395f0}, 0xc000863080?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000d0f400 sp=0xc000d0f3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc000e395f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000d0f490 sp=0xc000d0f400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0010a62d0}, 0xc000863080, 0xc000dd9380) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000d0f7d0 sp=0xc000d0f490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0010a62d0}, 0x3f252a0?, 0xc000813ef0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000d0f848 sp=0xc000d0f7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00094f440, 0xc000c70450, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000d0fdb0 sp=0xc000d0f848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00094f440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000d0ff28 sp=0xc000d0fdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000d0ffe0 sp=0xc000d0ff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d0ffe8 sp=0xc000d0ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 190 [select]: +runtime.gopark(0xc0012cca68?, 0x2?, 0x0?, 0x0?, 0xc0012cca14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012cc880 sp=0xc0012cc860 pc=0x4777ae +runtime.selectgo(0xc0012cca68, 0xc0012cca10, 0x7f429a410500?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012cc9d0 sp=0xc0012cc880 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000b76c80, {0xc000d194b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0012cca98 sp=0xc0012cc9d0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000b76c80, {0xc000d194b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0012ccb30 sp=0xc0012cca98 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000c71b00, {0xc000d194b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0012ccb78 sp=0xc0012ccb30 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000c71b00}, {0xc000d194b0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0012ccbc0 sp=0xc0012ccb78 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000e51560, {0xc000d194b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0012ccc30 sp=0xc0012ccbc0 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000d194a0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0012ccd00 sp=0xc0012ccc30 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc000e51560, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0012ccdf0 sp=0xc0012ccd00 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000d194a0, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43dd860, 0xc0016634f0}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0012ccea8 sp=0xc0012ccdf0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0009ecff0, {0x43dd860?, 0xc0016634f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0012cd0e0 sp=0xc0012ccea8 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc000d195f0, {0x43dd860, 0xc0016634f0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0012cd1c8 sp=0xc0012cd0e0 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000ee6a80, {0x43dd860, 0xc0016634f0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0012cd200 sp=0xc0012cd1c8 pc=0x1fd94b2 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamServer).Recv(0xc000b6fe40) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:167 +0x66 fp=0xc0012cd240 sp=0xc0012cd200 pc=0x2102886 +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream(0xc000c4a630, {0x5b512b8, 0xc000b6fe40}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:87 +0x218 fp=0xc0012cd370 sp=0xc0012cd240 pc=0x2154718 +github.com/hashicorp/go-plugin/internal/plugin._GRPCBroker_StartStream_Handler({0x41cb860?, 0xc000c4a630}, {0x5b4a6f0?, 0xc000ee6a80}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:148 +0xb6 fp=0xc0012cd3b0 sp=0xc0012cd370 pc=0x2102716 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41cb860, 0xc000c4a630}, {0x5b4ab30, 0xc000d195f0}, 0xc000f1ed68?, 0x47efc98) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0012cd400 sp=0xc0012cd3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41cb860, 0xc000c4a630}, {0x5b4ab30, 0xc000d195f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0012cd490 sp=0xc0012cd400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41cb860, 0xc000c4a630}, {0x5b492f8, 0xc0009ecff0}, 0xc000f1ed68, 0xc000fe72c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0012cd7d0 sp=0xc0012cd490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41cb860, 0xc000c4a630}, {0x5b492f8, 0xc0009ecff0}, 0x3f252a0?, 0xc000b6f870?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0012cd848 sp=0xc0012cd7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc000e51560, 0xc000c4a6c0, 0x770cb80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0012cddb0 sp=0xc0012cd848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc000e51560, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0012cdf28 sp=0xc0012cddb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0012cdfe0 sp=0xc0012cdf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012cdfe8 sp=0xc0012cdfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 198 [select]: +runtime.gopark(0xc0010c8a68?, 0x2?, 0x0?, 0x0?, 0xc0010c8a14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010c8880 sp=0xc0010c8860 pc=0x4777ae +runtime.selectgo(0xc0010c8a68, 0xc0010c8a10, 0x7f429a410200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010c89d0 sp=0xc0010c8880 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0000fd130, {0xc000eb8580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0010c8a98 sp=0xc0010c89d0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0000fd130, {0xc000eb8580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0010c8b30 sp=0xc0010c8a98 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000eb84b0, {0xc000eb8580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0010c8b78 sp=0xc0010c8b30 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000eb84b0}, {0xc000eb8580, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0010c8bc0 sp=0xc0010c8b78 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00094f560, {0xc000eb8580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0010c8c30 sp=0xc0010c8bc0 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000eb8570, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0010c8d00 sp=0xc0010c8c30 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc00094f560, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0010c8df0 sp=0xc0010c8d00 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000eb8570, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43dd860, 0xc0011f25f0}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0010c8ea8 sp=0xc0010c8df0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000e840f0, {0x43dd860?, 0xc0011f25f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0010c90e0 sp=0xc0010c8ea8 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc000eb86c0, {0x43dd860, 0xc0011f25f0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0010c91c8 sp=0xc0010c90e0 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0005fb2c0, {0x43dd860, 0xc0011f25f0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0010c9200 sp=0xc0010c91c8 pc=0x1fd94b2 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamServer).Recv(0xc000e9c580) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:167 +0x66 fp=0xc0010c9240 sp=0xc0010c9200 pc=0x2102886 +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream(0xc000c70510, {0x5b512b8, 0xc000e9c580}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:87 +0x218 fp=0xc0010c9370 sp=0xc0010c9240 pc=0x2154718 +github.com/hashicorp/go-plugin/internal/plugin._GRPCBroker_StartStream_Handler({0x41cb860?, 0xc000c70510}, {0x5b4a6f0?, 0xc0005fb2c0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:148 +0xb6 fp=0xc0010c93b0 sp=0xc0010c9370 pc=0x2102716 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41cb860, 0xc000c70510}, {0x5b4ab30, 0xc000eb86c0}, 0xc000a724f8?, 0x47efc98) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0010c9400 sp=0xc0010c93b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41cb860, 0xc000c70510}, {0x5b4ab30, 0xc000eb86c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0010c9490 sp=0xc0010c9400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41cb860, 0xc000c70510}, {0x5b492f8, 0xc000e840f0}, 0xc000a724f8, 0xc0008c4a80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0010c97d0 sp=0xc0010c9490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41cb860, 0xc000c70510}, {0x5b492f8, 0xc000e840f0}, 0x3f252a0?, 0xc000e9c4b0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0010c9848 sp=0xc0010c97d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00094f560, 0xc000c705a0, 0x770cb80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0010c9db0 sp=0xc0010c9848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00094f560, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0010c9f28 sp=0xc0010c9db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0010c9fe0 sp=0xc0010c9f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010c9fe8 sp=0xc0010c9fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 199 [select]: +runtime.gopark(0xc000efcfa0?, 0x3?, 0x0?, 0x4e?, 0xc000efcf3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000efcda8 sp=0xc000efcd88 pc=0x4777ae +runtime.selectgo(0xc000efcfa0, 0xc000efcf34, 0xc000efcf40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000efcef8 sp=0xc000efcda8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:73 +0x1ad fp=0xc000efcfe0 sp=0xc000efcef8 pc=0x2154aad +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000efcfe8 sp=0xc000efcfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream in goroutine 198 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:71 +0x1d1 + +goroutine 211 [select]: +runtime.gopark(0xc000efdfa0?, 0x3?, 0x0?, 0x0?, 0xc000efdf3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000efdda8 sp=0xc000efdd88 pc=0x4777ae +runtime.selectgo(0xc000efdfa0, 0xc000efdf34, 0xc000efdf40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000efdef8 sp=0xc000efdda8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:73 +0x1ad fp=0xc000efdfe0 sp=0xc000efdef8 pc=0x2154aad +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000efdfe8 sp=0xc000efdfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream in goroutine 190 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:71 +0x1d1 + +goroutine 191 [select]: +runtime.gopark(0xc001096fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001096f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001096de8 sp=0xc001096dc8 pc=0x4777ae +runtime.selectgo(0xc001096fb0, 0xc001096f70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001096f38 sp=0xc001096de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001096fe0 sp=0xc001096f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001096fe8 sp=0xc001096fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 192 [select]: +runtime.gopark(0xc0010aefb0?, 0x2?, 0xb0?, 0xed?, 0xc0010aeee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010aed58 sp=0xc0010aed38 pc=0x4777ae +runtime.selectgo(0xc0010aefb0, 0xc0010aeee0, 0x448855?, 0x0, 0x4166b20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010aeea8 sp=0xc0010aed58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0010aefe0 sp=0xc0010aeea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010aefe8 sp=0xc0010aefe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 193 [chan receive]: +runtime.gopark(0x7700000000000000?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001097d60 sp=0xc001097d40 pc=0x4777ae +runtime.chanrecv(0xc000c6b1a0, 0xc001097e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001097dd8 sp=0xc001097d60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001097e00 sp=0xc001097dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001097fe0 sp=0xc001097e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001097fe8 sp=0xc001097fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 626 [chan receive]: +runtime.gopark(0xc000912b88?, 0x4b05a9?, 0x28?, 0x2f?, 0x5d54ac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001242ee8 sp=0xc001242ec8 pc=0x4777ae +runtime.chanrecv(0xc000f26540, 0xc001242fa0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001242f60 sp=0xc001242ee8 pc=0x441325 +runtime.chanrecv2(0x4ad421?, 0xc001242fd0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc001242f88 sp=0xc001242f60 pc=0x440f92 +google.golang.org/grpc.(*Server).serverWorker(0xc001ebf4a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:593 +0x85 fp=0xc001242fc0 sp=0xc001242f88 pc=0x11744a5 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc001242fe0 sp=0xc001242fc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001242fe8 sp=0xc001242fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 559 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 201 [select]: +runtime.gopark(0xc0010abfb0?, 0x2?, 0xb4?, 0x1b?, 0xc0010abf74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010abde8 sp=0xc0010abdc8 pc=0x4777ae +runtime.selectgo(0xc0010abfb0, 0xc0010abf70, 0x2154a5f?, 0x0, 0xc000e9c580?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010abf38 sp=0xc0010abde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0010abfe0 sp=0xc0010abf38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010abfe8 sp=0xc0010abfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 202 [select]: +runtime.gopark(0xc0010acfb0?, 0x2?, 0xb0?, 0xcd?, 0xc0010acee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010acd58 sp=0xc0010acd38 pc=0x4777ae +runtime.selectgo(0xc0010acfb0, 0xc0010acee0, 0x280487a?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010acea8 sp=0xc0010acd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0010acfe0 sp=0xc0010acea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010acfe8 sp=0xc0010acfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 203 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010aad60 sp=0xc0010aad40 pc=0x4777ae +runtime.chanrecv(0xc000c9da40, 0xc0010aae60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0010aadd8 sp=0xc0010aad60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0010aae00 sp=0xc0010aadd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0010aafe0 sp=0xc0010aae00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010aafe8 sp=0xc0010aafe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 226 [select]: +runtime.gopark(0xc000f02890?, 0x2?, 0x4e?, 0x1a?, 0xc000f0283c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f026a8 sp=0xc000f02688 pc=0x4777ae +runtime.selectgo(0xc000f02890, 0xc000f02838, 0x7f429a416200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f027f8 sp=0xc000f026a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000b77b30, {0xc00110a430, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc000f028c0 sp=0xc000f027f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000b77b30, {0xc00110a430, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc000f02958 sp=0xc000f028c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00110a360, {0xc00110a430, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc000f029a0 sp=0xc000f02958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc00110a360}, {0xc00110a430, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000f029e8 sp=0xc000f029a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000e51d40, {0xc00110a430, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc000f02a58 sp=0xc000f029e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc00110a420, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc000f02b28 sp=0xc000f02a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001109440?, 0xc000e51d40, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc000f02c18 sp=0xc000f02b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc00110a420, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc000b77f90}, 0xc000c70240?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc000f02cd0 sp=0xc000f02c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000c072c0, {0x43dcf60?, 0xc000b77f90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc000f02f08 sp=0xc000f02cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc00110a570, {0x43dcf60, 0xc000b77f90}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc000f02ff0 sp=0xc000f02f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000b75a80, {0x43dcf60, 0xc000b77f90}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc000f03028 sp=0xc000f02ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc000c6f0b0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc000f03068 sp=0xc000f03028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc000c6f0b0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc000f030a8 sp=0xc000f03068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc000c6f0c0) + :1 +0x46 fp=0xc000f030e0 sp=0xc000f030a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c703c0, {0x5b51368, 0xc000c6f0c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc000f03320 sp=0xc000f030e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000c6e5e0, {0x5b51038, 0xc000c6f0b0}) + :1 +0xaf fp=0xc000f03370 sp=0xc000f03320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000c6e5e0}, {0x5b4a6f0?, 0xc000b75a80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc000f033b0 sp=0xc000f03370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc00110a570}, 0xc000dc24c8?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000f03400 sp=0xc000f033b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc00110a570}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000f03490 sp=0xc000f03400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc000c072c0}, 0xc000dc24c8, 0xc001108840) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000f037d0 sp=0xc000f03490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc000c072c0}, 0x3f252a0?, 0xc000c6efe0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000f03848 sp=0xc000f037d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc000e51d40, 0xc000c70450, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000f03db0 sp=0xc000f03848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc000e51d40, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000f03f28 sp=0xc000f03db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000f03fe0 sp=0xc000f03f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f03fe8 sp=0xc000f03fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 627 [chan receive]: +runtime.gopark(0x0?, 0x4?, 0x51?, 0x80?, 0xc001f16898?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010afee8 sp=0xc0010afec8 pc=0x4777ae +runtime.chanrecv(0xc000f26540, 0xc0010affa0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0010aff60 sp=0xc0010afee8 pc=0x441325 +runtime.chanrecv2(0x1?, 0x1?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0010aff88 sp=0xc0010aff60 pc=0x440f92 +google.golang.org/grpc.(*Server).serverWorker(0xc001ebf4a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:593 +0x85 fp=0xc0010affc0 sp=0xc0010aff88 pc=0x11744a5 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc0010affe0 sp=0xc0010affc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010affe8 sp=0xc0010affe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 559 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 204 [select]: +runtime.gopark(0xc000f06890?, 0x2?, 0x98?, 0x67?, 0xc000f0683c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f066a8 sp=0xc000f06688 pc=0x4777ae +runtime.selectgo(0xc000f06890, 0xc000f06838, 0x7f429a415c00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f067f8 sp=0xc000f066a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0008302d0, {0xc000eb8c40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc000f068c0 sp=0xc000f067f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0008302d0, {0xc000eb8c40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc000f06958 sp=0xc000f068c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000eb8b70, {0xc000eb8c40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc000f069a0 sp=0xc000f06958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000eb8b70}, {0xc000eb8c40, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000f069e8 sp=0xc000f069a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00094f9e0, {0xc000eb8c40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc000f06a58 sp=0xc000f069e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000eb8c30, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc000f06b28 sp=0xc000f06a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc000ed0a80?, 0xc00094f9e0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc000f06c18 sp=0xc000f06b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000eb8c30, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc000830690}, 0xc000c4a360?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc000f06cd0 sp=0xc000f06c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000e841e0, {0x43dcf60?, 0xc000830690}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc000f06f08 sp=0xc000f06cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc000eb8d80, {0x43dcf60, 0xc000830690}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc000f06ff0 sp=0xc000f06f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0005fb480, {0x43dcf60, 0xc000830690}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc000f07028 sp=0xc000f06ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc000e9c8b0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc000f07068 sp=0xc000f07028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc000e9c8b0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc000f070a8 sp=0xc000f07068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc000e9c8c0) + :1 +0x46 fp=0xc000f070e0 sp=0xc000f070a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c4a4e0, {0x5b51368, 0xc000e9c8c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc000f07320 sp=0xc000f070e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000bfd440, {0x5b51038, 0xc000e9c8b0}) + :1 +0xaf fp=0xc000f07370 sp=0xc000f07320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000bfd440}, {0x5b4a6f0?, 0xc0005fb480}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc000f073b0 sp=0xc000f07370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc000eb8d80}, 0xc000a726f0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000f07400 sp=0xc000f073b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc000eb8d80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000f07490 sp=0xc000f07400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000e841e0}, 0xc000a726f0, 0xc000ed0080) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000f077d0 sp=0xc000f07490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000e841e0}, 0x3f252a0?, 0xc000e9c7e0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000f07848 sp=0xc000f077d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc00094f9e0, 0xc000c4a570, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000f07db0 sp=0xc000f07848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc00094f9e0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000f07f28 sp=0xc000f07db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000f07fe0 sp=0xc000f07f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f07fe8 sp=0xc000f07fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 513 [select]: +runtime.gopark(0xc001244f40?, 0x3?, 0xfd?, 0xfd?, 0xc001244d2a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001244b60 sp=0xc001244b40 pc=0x4777ae +runtime.selectgo(0xc001244f40, 0xc001244d24, 0xc001244f70?, 0x0, 0x48e2a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001244cb0 sp=0xc001244b60 pc=0x48916b +github.com/rancher/opni/pkg/health.(*Monitor).WatchHealthStatus.func1() + /home/ec2-user/opni/pkg/health/monitor.go:166 +0x68e fp=0xc001244fe0 sp=0xc001244cb0 pc=0x207082e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001244fe8 sp=0xc001244fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/health.(*Monitor).WatchHealthStatus in goroutine 573 + /home/ec2-user/opni/pkg/health/monitor.go:150 +0x8da + +goroutine 236 [select]: +runtime.gopark(0xc001134890?, 0x2?, 0x98?, 0x47?, 0xc00113483c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011346a8 sp=0xc001134688 pc=0x4777ae +runtime.selectgo(0xc001134890, 0xc001134838, 0x7f429a415600?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011347f8 sp=0xc0011346a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00099b950, {0xc00110b780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0011348c0 sp=0xc0011347f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00099b950, {0xc00110b780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001134958 sp=0xc0011348c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00110af30, {0xc00110b780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0011349a0 sp=0xc001134958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc00110af30}, {0xc00110b780, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0011349e8 sp=0xc0011349a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00110e6c0, {0xc00110b780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001134a58 sp=0xc0011349e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc00110b770, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001134b28 sp=0xc001134a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc00112b000?, 0xc00110e6c0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001134c18 sp=0xc001134b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc00110b770, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc00010f6d0}, 0xc000c70240?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001134cd0 sp=0xc001134c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000c073b0, {0x43dcf60?, 0xc00010f6d0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001134f08 sp=0xc001134cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc00110b8c0, {0x43dcf60, 0xc00010f6d0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001134ff0 sp=0xc001134f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000b75f20, {0x43dcf60, 0xc00010f6d0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001135028 sp=0xc001134ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc000c6f940) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001135068 sp=0xc001135028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc000c6f940?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0011350a8 sp=0xc001135068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc000c6f950) + :1 +0x46 fp=0xc0011350e0 sp=0xc0011350a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c703c0, {0x5b51368, 0xc000c6f950}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001135320 sp=0xc0011350e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000c6e5e0, {0x5b51038, 0xc000c6f940}) + :1 +0xaf fp=0xc001135370 sp=0xc001135320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000c6e5e0}, {0x5b4a6f0?, 0xc000b75f20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0011353b0 sp=0xc001135370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc00110b8c0}, 0xc000dc2c30?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001135400 sp=0xc0011353b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc00110b8c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001135490 sp=0xc001135400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc000c073b0}, 0xc000dc2c30, 0xc00112a400) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0011357d0 sp=0xc001135490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc000c073b0}, 0x3f252a0?, 0xc000c6f870?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001135848 sp=0xc0011357d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00110e6c0, 0xc000c70450, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001135db0 sp=0xc001135848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00110e6c0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001135f28 sp=0xc001135db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001135fe0 sp=0xc001135f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001135fe8 sp=0xc001135fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 216 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x20?, 0x75?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001187cd0 sp=0xc001187cb0 pc=0x4777ae +runtime.chanrecv(0xc000fa56e0, 0xc001187de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001187d48 sp=0xc001187cd0 pc=0x441325 +runtime.chanrecv1(0xc001187df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001187d70 sp=0xc001187d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001187f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001187e00 sp=0xc001187d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc000bdd540, 0x420c60?, 0xc000d19770) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001187fa0 sp=0xc001187e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001187fe0 sp=0xc001187fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001187fe8 sp=0xc001187fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 213 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 230 [select]: +runtime.gopark(0xc001110fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001110f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001110de8 sp=0xc001110dc8 pc=0x4777ae +runtime.selectgo(0xc001110fb0, 0xc001110f70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001110f38 sp=0xc001110de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001110fe0 sp=0xc001110f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001110fe8 sp=0xc001110fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 231 [select]: +runtime.gopark(0xc001116fb0?, 0x2?, 0xb0?, 0x6d?, 0xc001116ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001116d58 sp=0xc001116d38 pc=0x4777ae +runtime.selectgo(0xc001116fb0, 0xc001116ee0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001116ea8 sp=0xc001116d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001116fe0 sp=0xc001116ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001116fe8 sp=0xc001116fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 232 [chan receive]: +runtime.gopark(0x5000000000000000?, 0x0?, 0x1?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001098d60 sp=0xc001098d40 pc=0x4777ae +runtime.chanrecv(0xc000c6b500, 0xc001098e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001098dd8 sp=0xc001098d60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001098e00 sp=0xc001098dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001098fe0 sp=0xc001098e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001098fe8 sp=0xc001098fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 628 [chan receive]: +runtime.gopark(0xc000912b80?, 0x2182b94?, 0x78?, 0xcf?, 0x24c59ff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012acee8 sp=0xc0012acec8 pc=0x4777ae +runtime.chanrecv(0xc000f26540, 0xc0012acfa0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0012acf60 sp=0xc0012acee8 pc=0x441325 +runtime.chanrecv2(0x4ad421?, 0xc0012acfd0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0012acf88 sp=0xc0012acf60 pc=0x440f92 +google.golang.org/grpc.(*Server).serverWorker(0xc001ebf4a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:593 +0x85 fp=0xc0012acfc0 sp=0xc0012acf88 pc=0x11744a5 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc0012acfe0 sp=0xc0012acfc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012acfe8 sp=0xc0012acfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 559 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 262 [chan receive]: +runtime.gopark(0x4be6c0?, 0xc000c4b2c0?, 0x20?, 0xe7?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001146f08 sp=0xc001146ee8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001146f80 sp=0xc001146f08 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0x5b11a20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001146fa8 sp=0xc001146f80 pc=0x440f72 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseNodeManagerClient(0xc0005a8c80, {0x5b11a20, 0xc000fe12f0}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:222 +0x93 fp=0xc001146fd0 sp=0xc001146fa8 pc=0x3972293 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseNodeManagerClient(0xc00091c300, {0x4b05a9?, 0xc0012130f0?}, 0xc0010bb410) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:113 +0x18d fp=0xc001147098 sp=0xc001146fd0 pc=0x21a68cd +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler.func1({0x5b426a8, 0xc0010bb4d0}, {0x4278de0?, 0xc0010bb410}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:169 +0x83 fp=0xc0011470f0 sp=0xc001147098 pc=0x21ac443 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc0010bb4d0}, {0x4278de0, 0xc0010bb410}, 0xc000997aa0, 0xc000863e78) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001147148 sp=0xc0011470f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc0010bb4d0}, {0x4278de0, 0xc0010bb410}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc0011471e8 sp=0xc001147148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc0010bb3e0}, {0x4278de0, 0xc0010bb410}, 0xc000997aa0, 0xc0010b6b80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc0011476b8 sp=0xc0011471e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc0010bb3e0}, {0x4278de0, 0xc0010bb410}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001147740 sp=0xc0011476b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler({0x4278ea0?, 0xc00091c300}, {0x5b426a8, 0xc0010bb3e0}, 0xc0004b70a0, 0xc000b75440) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:171 +0x1c4 fp=0xc0011477d0 sp=0xc001147740 pc=0x21ac2a4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc000f8b8c0, 0xc000c70f60, 0x771ecb8, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001147db0 sp=0xc0011477d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc000f8b8c0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001147f28 sp=0xc001147db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001147fe0 sp=0xc001147f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001147fe8 sp=0xc001147fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 296 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xe0?, 0x7b?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010a1cd0 sp=0xc0010a1cb0 pc=0x4777ae +runtime.chanrecv(0xc0012485a0, 0xc0010a1de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0010a1d48 sp=0xc0010a1cd0 pc=0x441325 +runtime.chanrecv1(0xc0010a1df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0010a1d70 sp=0xc0010a1d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc0010a1f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc0010a1e00 sp=0xc0010a1d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc000bdd540, 0x0?, 0xc000ff95c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc0010a1fa0 sp=0xc0010a1e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc0010a1fe0 sp=0xc0010a1fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010a1fe8 sp=0xc0010a1fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 294 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 206 [select]: +runtime.gopark(0xc001194fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001194f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001194de8 sp=0xc001194dc8 pc=0x4777ae +runtime.selectgo(0xc001194fb0, 0xc001194f70, 0x2154a5f?, 0x0, 0xc000e9c580?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001194f38 sp=0xc001194de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001194fe0 sp=0xc001194f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001194fe8 sp=0xc001194fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 207 [select]: +runtime.gopark(0xc001193fb0?, 0x2?, 0xb0?, 0x3d?, 0xc001193ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001193d58 sp=0xc001193d38 pc=0x4777ae +runtime.selectgo(0xc001193fb0, 0xc001193ee0, 0x472ddb?, 0x0, 0x2166b62?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001193ea8 sp=0xc001193d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001193fe0 sp=0xc001193ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001193fe8 sp=0xc001193fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 208 [chan receive]: +runtime.gopark(0x7700000000000000?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010add60 sp=0xc0010add40 pc=0x4777ae +runtime.chanrecv(0xc000c9dda0, 0xc0010ade60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0010addd8 sp=0xc0010add60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0010ade00 sp=0xc0010addd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0010adfe0 sp=0xc0010ade00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010adfe8 sp=0xc0010adfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 722 [chan receive]: +runtime.gopark(0x0?, 0x4b05a9?, 0x50?, 0x8e?, 0x4bff4a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019f8e18 sp=0xc0019f8df8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0019f8e90 sp=0xc0019f8e18 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x7f429a441700?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0019f8eb8 sp=0xc0019f8e90 pc=0x440f72 +github.com/rancher/opni/pkg/gateway.NewGateway.func5() + /home/ec2-user/opni/pkg/gateway/gateway.go:308 +0x5c fp=0xc0019f8f50 sp=0xc0019f8eb8 pc=0x22aea9c +github.com/rancher/opni/pkg/util/waitctx.restrictive.Go.func1() + /home/ec2-user/opni/pkg/util/waitctx/context.go:162 +0x93 fp=0xc0019f8fe0 sp=0xc0019f8f50 pc=0x178df73 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019f8fe8 sp=0xc0019f8fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/util/waitctx.restrictive.Go in goroutine 609 + /home/ec2-user/opni/pkg/util/waitctx/context.go:160 +0xeb + +goroutine 217 [select]: +runtime.gopark(0xc00120f448?, 0x2?, 0xd0?, 0xf2?, 0xc00120f3fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00120f278 sp=0xc00120f258 pc=0x4777ae +runtime.selectgo(0xc00120f448, 0xc00120f3f8, 0xc000b6efe8?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00120f3c8 sp=0xc00120f278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000926a20) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc00120f478 sp=0xc00120f3c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00094cf70, {0x42219c0?, 0xc000d19d10}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc00120f6d8 sp=0xc00120f478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000926894?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc00120f738 sp=0xc00120f6d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0009267e0, 0xc00120f8f0, 0xc00120f8c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc00120f7e0 sp=0xc00120f738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0009267e0, {0x42219c0, 0xc000d19d10}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc00120f920 sp=0xc00120f7e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc000d19dd0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc000d19ce0}, {0x42219c0, 0xc000d19d10}, 0x0?, {0xc0005fac80, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc00120f9c0 sp=0xc00120f920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc000bdcaa0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc000d19ce0}, {0x42219c0, 0xc000d19d10}, 0xc0009c7400, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc00120fd30 sp=0xc00120f9c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc0009c7400, {0x5b426e0, 0xc000bdcaa0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc000d19ce0}, {0x42219c0, 0xc000d19d10}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc00120fe58 sp=0xc00120fd30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseManagementAPI(0xc000c6f160, {0x5b426e0, 0xc000bdcaa0}, 0x0?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:51 +0xee fp=0xc00120fee8 sp=0xc00120fe58 pc=0x21ab66e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI.func2(0x1) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:190 +0xf2 fp=0xc00120ff58 sp=0xc00120fee8 pc=0x21a7f32 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc00120ffe0 sp=0xc00120ff58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00120ffe8 sp=0xc00120ffe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 213 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 286 [select]: +runtime.gopark(0xc0013f5e48?, 0x2?, 0x0?, 0x0?, 0xc0013f5dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f0dc30 sp=0xc001f0dc10 pc=0x4777ae +runtime.selectgo(0xc001f0de48, 0xc0013f5dc8, 0xc0013f5dd8?, 0x0, 0xc00125e000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f0dd80 sp=0xc001f0dc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139c190, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001f0de78 sp=0xc001f0dd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000573030) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001f0df30 sp=0xc001f0de78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc001f0dfe0 sp=0xc001f0df30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f0dfe8 sp=0xc001f0dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 304 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 213 [select]: +runtime.gopark(0xc001113f28?, 0x2?, 0xb4?, 0x1b?, 0xc001113ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001113d30 sp=0xc001113d10 pc=0x4777ae +runtime.selectgo(0xc001113f28, 0xc001113eb8, 0x0?, 0x0, 0xc001113f20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001113e80 sp=0xc001113d30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc00110a780, 0xc000f1ee28, 0xc000b6ffc0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001113f58 sp=0xc001113e80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI(0xc00110a780, {0x5b6f770?, 0xc000d8a000}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:185 +0x10a fp=0xc001113f98 sp=0xc001113f58 pc=0x21a7e0a +github.com/rancher/opni/pkg/management.NewServer.func1.2() + /home/ec2-user/opni/pkg/management/server.go:156 +0x62 fp=0xc001113fe0 sp=0xc001113f98 pc=0x231c0c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001113fe8 sp=0xc001113fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 212 + /home/ec2-user/opni/pkg/management/server.go:156 +0x137 + +goroutine 214 [select]: +runtime.gopark(0xc001111f28?, 0x2?, 0xb4?, 0x1b?, 0xc001111ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001111d30 sp=0xc001111d10 pc=0x4777ae +runtime.selectgo(0xc001111f28, 0xc001111eb8, 0x0?, 0x0, 0xc001111f20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001111e80 sp=0xc001111d30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc00110a780, 0xc000f1ee40, 0xc000fe0020) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001111f58 sp=0xc001111e80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer(0xc00110a780, {0x5b21b70?, 0xc000ef5240}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:198 +0x10a fp=0xc001111f98 sp=0xc001111f58 pc=0x21a810a +github.com/rancher/opni/pkg/management.NewServer.func1.3() + /home/ec2-user/opni/pkg/management/server.go:158 +0x62 fp=0xc001111fe0 sp=0xc001111f98 pc=0x231c022 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001111fe8 sp=0xc001111fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 212 + /home/ec2-user/opni/pkg/management/server.go:158 +0x245 + +goroutine 1192 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00172d9a8 sp=0xc00172d988 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00172d9e0 sp=0xc00172d9a8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d868, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00172da00 sp=0xc00172d9e0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc002083b20, 0x448801?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00172da48 sp=0xc00172da00 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc002083b00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc00172db38 sp=0xc00172da48 pc=0x513f65 +net.(*netFD).accept(0xc002083b00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc00172dc30 sp=0xc00172db38 pc=0x8967be +net.(*TCPListener).accept(0xc00127d0a0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc00172dc78 sp=0xc00172dc30 pc=0x8c383e +net.(*TCPListener).Accept(0xc00127d0a0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc00172dce8 sp=0xc00172dc78 pc=0x8c1ba5 +net/http.(*onceCloseListener).Accept(0xc00224ec90) + :1 +0x44 fp=0xc00172dd20 sp=0xc00172dce8 pc=0xab5544 +net/http.(*Server).Serve(0xc001388960, {0x5b2a6d0, 0xc00127d0a0}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc00172df18 sp=0xc00172dd20 pc=0xa7859f +net/http.(*Server).ListenAndServe(0xc001388960) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2985 +0xbd fp=0xc00172df70 sp=0xc00172df18 pc=0xa77f9d +github.com/rancher/opni/pkg/management.(*Server).listenAndServeHttp.func2() + /home/ec2-user/opni/pkg/management/server.go:246 +0x2f fp=0xc00172dfa0 sp=0xc00172df70 pc=0x231edef +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc00172dfe0 sp=0xc00172dfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00172dfe8 sp=0xc00172dfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1144 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 259 [select]: +runtime.gopark(0xc0010b0fb0?, 0x2?, 0x0?, 0xe?, 0xc0010b0f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010b0da8 sp=0xc0010b0d88 pc=0x4777ae +runtime.selectgo(0xc0010b0fb0, 0xc0010b0f28, 0x0?, 0x0, 0x13?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010b0ef8 sp=0xc0010b0da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc0010b0fe0 sp=0xc0010b0ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010b0fe8 sp=0xc0010b0fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 258 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 218 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x60?, 0x72?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001205cd0 sp=0xc001205cb0 pc=0x4777ae +runtime.chanrecv(0xc000fa54a0, 0xc001205de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001205d48 sp=0xc001205cd0 pc=0x441325 +runtime.chanrecv1(0xc001205df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001205d70 sp=0xc001205d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001205f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001205e00 sp=0xc001205d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc000bdd540, 0x2?, 0xc000d197d0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001205fa0 sp=0xc001205e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001205fe0 sp=0xc001205fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001205fe8 sp=0xc001205fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 214 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 219 [select]: +runtime.gopark(0xc001221448?, 0x2?, 0xd0?, 0x12?, 0xc0012213fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001221278 sp=0xc001221258 pc=0x4777ae +runtime.selectgo(0xc001221448, 0xc0012213f8, 0xc000b6efe8?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012213c8 sp=0xc001221278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000927440) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001221478 sp=0xc0012213c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00094d380, {0x42219c0?, 0xc000ff8570}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0012216d8 sp=0xc001221478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000926bf4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001221738 sp=0xc0012216d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc000926b40, 0xc0012218f0, 0xc0012218c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc0012217e0 sp=0xc001221738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc000926b40, {0x42219c0, 0xc000ff8570}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001221920 sp=0xc0012217e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc000ff8630}, {0x4673e80, 0x23}, {0x4278de0, 0xc000ff8540}, {0x42219c0, 0xc000ff8570}, 0x0?, {0xc0005fac80, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc0012219c0 sp=0xc001221920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc000bdcaa0}, {0x4673e80, 0x23}, {0x4278de0, 0xc000ff8540}, {0x42219c0, 0xc000ff8570}, 0xc0009c7400, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc001221d30 sp=0xc0012219c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc0009c7400, {0x5b426e0, 0xc000bdcaa0}, {0x4673e80, 0x23}, {0x4278de0, 0xc000ff8540}, {0x42219c0, 0xc000ff8570}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc001221e58 sp=0xc001221d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseNodeManagerClient(0xc000c6f160, {0x5b426e0, 0xc000bdcaa0}, 0xc000b80f50?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:60 +0xee fp=0xc001221ee8 sp=0xc001221e58 pc=0x21ab7ee +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer.func2(0x2) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:203 +0xf2 fp=0xc001221f58 sp=0xc001221ee8 pc=0x21a8232 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc001221fe0 sp=0xc001221f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001221fe8 sp=0xc001221fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 214 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 242 [select]: +runtime.gopark(0xc0011a1fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0011a1f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011a1de8 sp=0xc0011a1dc8 pc=0x4777ae +runtime.selectgo(0xc0011a1fb0, 0xc0011a1f70, 0xc000a53a40?, 0x0, 0xc000c9de00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011a1f38 sp=0xc0011a1de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0011a1fe0 sp=0xc0011a1f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011a1fe8 sp=0xc0011a1fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 209 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 243 [select]: +runtime.gopark(0xc0011a0fb0?, 0x2?, 0x0?, 0x0?, 0xc0011a0ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011a0d58 sp=0xc0011a0d38 pc=0x4777ae +runtime.selectgo(0xc0011a0fb0, 0xc0011a0ee0, 0x215759f?, 0x0, 0x2155d2b?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011a0ea8 sp=0xc0011a0d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0011a0fe0 sp=0xc0011a0ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011a0fe8 sp=0xc0011a0fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 209 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 244 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xe0?, 0x5d?, 0x22c5a46?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001195d60 sp=0xc001195d40 pc=0x4777ae +runtime.chanrecv(0xc00119a0c0, 0xc001195e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001195dd8 sp=0xc001195d60 pc=0x441325 +runtime.chanrecv1(0xc000e9c970?, 0xc00070f822?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001195e00 sp=0xc001195dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001195fe0 sp=0xc001195e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001195fe8 sp=0xc001195fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 209 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 245 [select]: +runtime.gopark(0xc001224890?, 0x2?, 0x98?, 0x47?, 0xc00122483c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012246a8 sp=0xc001224688 pc=0x4777ae +runtime.selectgo(0xc001224890, 0xc001224838, 0x7f429a419200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012247f8 sp=0xc0012246a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000831310, {0xc000ff8ac0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0012248c0 sp=0xc0012247f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000831310, {0xc000ff8ac0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001224958 sp=0xc0012248c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000eb96b0, {0xc000ff8ac0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0012249a0 sp=0xc001224958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000eb96b0}, {0xc000ff8ac0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0012249e8 sp=0xc0012249a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001196240, {0xc000ff8ac0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001224a58 sp=0xc0012249e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000ff8ab0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001224b28 sp=0xc001224a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001236580?, 0xc001196240, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001224c18 sp=0xc001224b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000ff8ab0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0007c01e0}, 0xc000c4a360?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001224cd0 sp=0xc001224c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001234000, {0x43dcf60?, 0xc0007c01e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001224f08 sp=0xc001224cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc000ff8c00, {0x43dcf60, 0xc0007c01e0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001224ff0 sp=0xc001224f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000ee75e0, {0x43dcf60, 0xc0007c01e0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001225028 sp=0xc001224ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc000fe0b60) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001225068 sp=0xc001225028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc000fe0b60?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0012250a8 sp=0xc001225068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc000fe0b70) + :1 +0x46 fp=0xc0012250e0 sp=0xc0012250a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c4a4e0, {0x5b51368, 0xc000fe0b70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001225320 sp=0xc0012250e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000bfd440, {0x5b51038, 0xc000fe0b60}) + :1 +0xaf fp=0xc001225370 sp=0xc001225320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000bfd440}, {0x5b4a6f0?, 0xc000ee75e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0012253b0 sp=0xc001225370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc000ff8c00}, 0xc000f1f4b8?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001225400 sp=0xc0012253b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc000ff8c00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001225490 sp=0xc001225400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc001234000}, 0xc000f1f4b8, 0xc000fe7b80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0012257d0 sp=0xc001225490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc001234000}, 0x3f252a0?, 0xc000fe0a90?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001225848 sp=0xc0012257d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc001196240, 0xc000c4a570, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001225db0 sp=0xc001225848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc001196240, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001225f28 sp=0xc001225db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001225fe0 sp=0xc001225f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001225fe8 sp=0xc001225fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 246 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x20?, 0xe7?, 0x30?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00114af98 sp=0xc00114af78 pc=0x4777ae +runtime.chanrecv(0xc0011a46c0, 0xc0011c5128, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00114b010 sp=0xc00114af98 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00114b038 sp=0xc00114b010 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/stream.(*gatewayStreamExtensionServerImpl).ConnectInternal(0xc000854620, {0x5b50ab0, 0xc000e9ce00}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/stream/plugin_gateway.go:208 +0xa4e fp=0xc00114b370 sp=0xc00114b038 pc=0x262088e +github.com/rancher/opni/pkg/plugins/apis/apiextensions._StreamAPIExtension_ConnectInternal_Handler({0x4229080?, 0xc000854620}, {0x5b4a6f0?, 0xc0005fb7c0}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go:284 +0xb6 fp=0xc00114b3b0 sp=0xc00114b370 pc=0x218ded6 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x4229080, 0xc000854620}, {0x5b4ab30, 0xc000eb99e0}, 0xc000a72c30?, 0x47ee568) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc00114b400 sp=0xc00114b3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x4229080, 0xc000854620}, {0x5b4ab30, 0xc000eb99e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc00114b490 sp=0xc00114b400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4229080, 0xc000854620}, {0x5b492f8, 0xc000e842d0}, 0xc000a72c30, 0xc000ed1540) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc00114b7d0 sp=0xc00114b490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x4229080, 0xc000854620}, {0x5b492f8, 0xc000e842d0}, 0x3f252a0?, 0xc000e9cd30?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc00114b848 sp=0xc00114b7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc001196360, 0xc000c4a9c0, 0x770c840, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc00114bdb0 sp=0xc00114b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc001196360, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc00114bf28 sp=0xc00114bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00114bfe0 sp=0xc00114bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00114bfe8 sp=0xc00114bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 239 [select]: +runtime.gopark(0xc001203f70?, 0x2?, 0xb4?, 0x1b?, 0xc001203ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001203d20 sp=0xc001203d00 pc=0x4777ae +runtime.selectgo(0xc001203f70, 0xc001203eb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001203e70 sp=0xc001203d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d2100, {0x5b426e0, 0xc00010f810}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001203fa0 sp=0xc001203e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001203fe0 sp=0xc001203fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001203fe8 sp=0xc001203fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 237 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 177 [chan receive]: +runtime.gopark(0x4b05a9?, 0xc0019c6e68?, 0x6d?, 0x2d?, 0xc0001277a0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001130dd8 sp=0xc001130db8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001130e50 sp=0xc001130dd8 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0x5b6f168?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001130e78 sp=0xc001130e50 pc=0x440f72 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI(0xc0005a8c80, {0x5b6f168, 0xc000fe1300}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:92 +0x619 fp=0xc001130fa0 sp=0xc001130e78 pc=0x396f019 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseManagementAPI(0xc00091c300, {0x4b05a9?, 0xc000a6b0f0?}, 0xc0010bb0e0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:102 +0x30f fp=0xc001131098 sp=0xc001130fa0 pc=0x21a640f +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler.func1({0x5b426a8, 0xc0010bb1a0}, {0x4278de0?, 0xc0010bb0e0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:151 +0x83 fp=0xc0011310f0 sp=0xc001131098 pc=0x21ac063 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc0010bb1a0}, {0x4278de0, 0xc0010bb0e0}, 0xc0009979c0, 0xc000863d28) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001131148 sp=0xc0011310f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc0010bb1a0}, {0x4278de0, 0xc0010bb0e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc0011311e8 sp=0xc001131148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc0010bb0b0}, {0x4278de0, 0xc0010bb0e0}, 0xc0009979c0, 0xc0010b6980) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc0011316b8 sp=0xc0011311e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc0010bb0b0}, {0x4278de0, 0xc0010bb0e0}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001131740 sp=0xc0011316b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler({0x4278ea0?, 0xc00091c300}, {0x5b426a8, 0xc0010bb0b0}, 0xc0004b6e70, 0xc000b75440) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:153 +0x1c4 fp=0xc0011317d0 sp=0xc001131740 pc=0x21abec4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc000f8b440, 0xc000c70f60, 0x771eca0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001131db0 sp=0xc0011317d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc000f8b440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001131f28 sp=0xc001131db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001131fe0 sp=0xc001131f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001131fe8 sp=0xc001131fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 247 [select]: +runtime.gopark(0xc001190fb0?, 0x2?, 0xa0?, 0xe?, 0xc001190f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001190da8 sp=0xc001190d88 pc=0x4777ae +runtime.selectgo(0xc001190fb0, 0xc001190f28, 0x4b0571?, 0x0, 0x11847d3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001190ef8 sp=0xc001190da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc001190fe0 sp=0xc001190ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001190fe8 sp=0xc001190fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 246 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 220 [chan receive]: +runtime.gopark(0x0?, 0x472ddb?, 0x71?, 0x5?, 0xc000bdcb08?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001112ef8 sp=0xc001112ed8 pc=0x4777ae +runtime.chanrecv(0xc000c9ccc0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001112f70 sp=0xc001112ef8 pc=0x441325 +runtime.chanrecv1(0xc000bdcaa0?, 0xc000e51440?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001112f98 sp=0xc001112f70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001112fe0 sp=0xc001112f98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001112fe8 sp=0xc001112fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 218 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 221 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001207978 sp=0xc001207958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0012079b0 sp=0xc001207978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677cc0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0012079d0 sp=0xc0012079b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d16920, 0xc001207a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001207a18 sp=0xc0012079d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000d16900) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001207b08 sp=0xc001207a18 pc=0x513f65 +net.(*netFD).accept(0xc000d16900) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001207c00 sp=0xc001207b08 pc=0x8967be +net.(*UnixListener).accept(0xc000d19800) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001207c40 sp=0xc001207c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc000d19800) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001207cb0 sp=0xc001207c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc000ee6de0) + :1 +0x44 fp=0xc001207ce8 sp=0xc001207cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc0004ef680, {0x5b2c6a0, 0xc000ee6de0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001207f18 sp=0xc001207ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc001207f68 sp=0xc001207f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc000ee71e0?, 0xc000fe06d0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001207fa8 sp=0xc001207f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001207fe0 sp=0xc001207fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001207fe8 sp=0xc001207fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 218 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 248 [select]: +runtime.gopark(0xc0011a3f90?, 0x2?, 0xb4?, 0x1b?, 0xc0011a3eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011a3d28 sp=0xc0011a3d08 pc=0x4777ae +runtime.selectgo(0xc0011a3f90, 0xc0011a3ea8, 0x4b0571?, 0x0, 0x5d8cc5?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011a3e78 sp=0xc0011a3d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc000001800) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc0011a3fc0 sp=0xc0011a3e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc0011a3fe0 sp=0xc0011a3fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011a3fe8 sp=0xc0011a3fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 246 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 222 [select]: +runtime.gopark(0xc00118ef38?, 0x2?, 0xc0?, 0x19?, 0xc00118ef14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00118ed90 sp=0xc00118ed70 pc=0x4777ae +runtime.selectgo(0xc00118ef38, 0xc00118ef10, 0xc00118ef18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00118eee0 sp=0xc00118ed90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc00118ef68 sp=0xc00118eee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc000f1f2a8?, 0xc000fe06f0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc00118efa8 sp=0xc00118ef68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc00118efe0 sp=0xc00118efa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00118efe8 sp=0xc00118efe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 218 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 249 [select]: +runtime.gopark(0xc0011fff40?, 0x2?, 0xb4?, 0x1b?, 0xc0011ffecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001497d48 sp=0xc001497d28 pc=0x4777ae +runtime.selectgo(0xc001497f40, 0xc0011ffec8, 0xc000067078?, 0x0, 0xc000bdc6f8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001497e98 sp=0xc001497d48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc000831900}, 0xc000001890, 0x0, 0xc0011a41e0, 0xc000e9cec0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc001497f70 sp=0xc001497e98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc001497fe0 sp=0xc001497f70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001497fe8 sp=0xc001497fe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 246 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 258 [chan receive]: +runtime.gopark(0x400a060?, 0xc00136dff0?, 0xe3?, 0x45?, 0x7?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00155bb28 sp=0xc00155bb08 pc=0x4777ae +runtime.chanrecv(0xc0011a4cc0, 0xc00120bcb8, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00155bba0 sp=0xc00155bb28 pc=0x441325 +runtime.chanrecv1(0xc0011c60d0?, 0x771d460?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00155bbc8 sp=0xc00155bba0 pc=0x440f72 +github.com/rancher/opni/pkg/gateway.(*StreamServer).OnPluginLoad.func1() + /home/ec2-user/opni/pkg/gateway/stream.go:285 +0x8eb fp=0xc00155bfe0 sp=0xc00155bbc8 pc=0x22bfbeb +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00155bfe8 sp=0xc00155bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.(*StreamServer).OnPluginLoad in goroutine 209 + /home/ec2-user/opni/pkg/gateway/stream.go:255 +0xa52 + +goroutine 260 [select]: +runtime.gopark(0xc0010b1f90?, 0x2?, 0xff?, 0xff?, 0xc0010b1eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010b1d28 sp=0xc0010b1d08 pc=0x4777ae +runtime.selectgo(0xc0010b1f90, 0xc0010b1ea8, 0x4b0571?, 0x0, 0xc000b85718?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010b1e78 sp=0xc0010b1d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc000f24780) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc0010b1fc0 sp=0xc0010b1e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc0010b1fe0 sp=0xc0010b1fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010b1fe8 sp=0xc0010b1fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 258 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 223 [chan receive]: +runtime.gopark(0x22c2125?, 0xc000b80f50?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00118fef8 sp=0xc00118fed8 pc=0x4777ae +runtime.chanrecv(0xc000c9ccc0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00118ff70 sp=0xc00118fef8 pc=0x441325 +runtime.chanrecv1(0xc000bdcaa0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00118ff98 sp=0xc00118ff70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc00118ffe0 sp=0xc00118ff98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00118ffe8 sp=0xc00118ffe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 216 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 261 [select]: +runtime.gopark(0xc0011e9f40?, 0x2?, 0xb4?, 0x1b?, 0xc0011e9ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00155dd48 sp=0xc00155dd28 pc=0x4777ae +runtime.selectgo(0xc00155df40, 0xc0011e9ec8, 0xc00110a030?, 0x0, 0x448855?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00155de98 sp=0xc00155dd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc0009d76d0}, 0xc000f24810, 0x0, 0xc0010bc6c0, 0xc0010b4af0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc00155df70 sp=0xc00155de98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc00155dfe0 sp=0xc00155df70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00155dfe8 sp=0xc00155dfe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 258 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 224 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001217978 sp=0xc001217958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0012179b0 sp=0xc001217978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677db8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0012179d0 sp=0xc0012179b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d167a0, 0xc001217a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001217a18 sp=0xc0012179d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000d16780) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001217b08 sp=0xc001217a18 pc=0x513f65 +net.(*netFD).accept(0xc000d16780) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001217c00 sp=0xc001217b08 pc=0x8967be +net.(*UnixListener).accept(0xc000d197a0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001217c40 sp=0xc001217c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc000d197a0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001217cb0 sp=0xc001217c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc000ee6d40) + :1 +0x44 fp=0xc001217ce8 sp=0xc001217cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc0004ef860, {0x5b2c6a0, 0xc000ee6d40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001217f18 sp=0xc001217ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc001217f68 sp=0xc001217f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc000ee74a0?, 0xc000fe0940?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001217fa8 sp=0xc001217f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001217fe0 sp=0xc001217fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001217fe8 sp=0xc001217fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 216 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 225 [select]: +runtime.gopark(0xc000ef7f38?, 0x2?, 0x2?, 0x0?, 0xc000ef7f14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000ef7d90 sp=0xc000ef7d70 pc=0x4777ae +runtime.selectgo(0xc000ef7f38, 0xc000ef7f10, 0xc000ef7f18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000ef7ee0 sp=0xc000ef7d90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc000ef7f68 sp=0xc000ef7ee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc000f1f440?, 0xc000fe0960?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc000ef7fa8 sp=0xc000ef7f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc000ef7fe0 sp=0xc000ef7fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000ef7fe8 sp=0xc000ef7fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 216 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 240 [select]: +runtime.gopark(0xc001201f70?, 0x2?, 0xb4?, 0x1b?, 0xc001201ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001201d20 sp=0xc001201d00 pc=0x4777ae +runtime.selectgo(0xc001201f70, 0xc001201eb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001201e70 sp=0xc001201d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d2160, {0x5b426e0, 0xc00010f860}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001201fa0 sp=0xc001201e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001201fe0 sp=0xc001201fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001201fe8 sp=0xc001201fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 237 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 424 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x80?, 0xde?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001557cd0 sp=0xc001557cb0 pc=0x4777ae +runtime.chanrecv(0xc00127bc80, 0xc001557de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001557d48 sp=0xc001557cd0 pc=0x441325 +runtime.chanrecv1(0xc001557df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001557d70 sp=0xc001557d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001557f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001557e00 sp=0xc001557d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc000ee4410, 0x420c60?, 0xc0015de210) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001557fa0 sp=0xc001557e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001557fe0 sp=0xc001557fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001557fe8 sp=0xc001557fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 420 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 250 [chan receive]: +runtime.gopark(0xc0008313b0?, 0x4b05a9?, 0x50?, 0xce?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011dce20 sp=0xc0011dce00 pc=0x4777ae +runtime.chanrecv(0xc00119a240, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0011dce98 sp=0xc0011dce20 pc=0x441325 +runtime.chanrecv1(0xc000eb99b0?, 0x48a0bb?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0011dcec0 sp=0xc0011dce98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0011dcfe0 sp=0xc0011dcec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011dcfe8 sp=0xc0011dcfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 246 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 251 [chan receive]: +runtime.gopark(0xc0008313b0?, 0x4b05a9?, 0x50?, 0xde?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011dde20 sp=0xc0011dde00 pc=0x4777ae +runtime.chanrecv(0xc00119a240, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0011dde98 sp=0xc0011dde20 pc=0x441325 +runtime.chanrecv1(0xc000eb99b0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0011ddec0 sp=0xc0011dde98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0011ddfe0 sp=0xc0011ddec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011ddfe8 sp=0xc0011ddfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 246 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 252 [select]: +runtime.gopark(0xc001183aa8?, 0x2?, 0x67?, 0x6a?, 0xc0011839c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001183838 sp=0xc001183818 pc=0x4777ae +runtime.selectgo(0xc001183aa8, 0xc0011839c0, 0xc0011839d0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001183988 sp=0xc001183838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc000be0aa0) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc001183ad8 sp=0xc001183988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc001196480, {0x5b426a8, 0xc000eb99b0}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc001183cf8 sp=0xc001183ad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc001183fe0 sp=0xc001183cf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001183fe8 sp=0xc001183fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 246 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 253 [chan receive]: +runtime.gopark(0x70?, 0x0?, 0x70?, 0x0?, 0xc000c9df80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011dee20 sp=0xc0011dee00 pc=0x4777ae +runtime.chanrecv(0xc000c9df80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0011dee98 sp=0xc0011dee20 pc=0x441325 +runtime.chanrecv1(0xc000eb9470?, 0x44063c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0011deec0 sp=0xc0011dee98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0011defe0 sp=0xc0011deec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011defe8 sp=0xc0011defe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 258 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 254 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc000c9df80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011dfe20 sp=0xc0011dfe00 pc=0x4777ae +runtime.chanrecv(0xc000c9df80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0011dfe98 sp=0xc0011dfe20 pc=0x441325 +runtime.chanrecv1(0xc000eb9470?, 0x4aead9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0011dfec0 sp=0xc0011dfe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0011dffe0 sp=0xc0011dfec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011dffe8 sp=0xc0011dffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 258 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 285 [select]: +runtime.gopark(0xc001433e70?, 0x2?, 0x0?, 0x0?, 0xc001433df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001de1c58 sp=0xc001de1c38 pc=0x4777ae +runtime.selectgo(0xc001de1e70, 0xc001433df0, 0xc001433e00?, 0x0, 0xc0010b8a80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001de1da8 sp=0xc001de1c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139c0a0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001de1ea0 sp=0xc001de1da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000572d90) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001de1f58 sp=0xc001de1ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc001de1fe0 sp=0xc001de1f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001de1fe8 sp=0xc001de1fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 322 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 289 [select]: +runtime.gopark(0xc002607e48?, 0x2?, 0x0?, 0x0?, 0xc002607dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002607c30 sp=0xc002607c10 pc=0x4777ae +runtime.selectgo(0xc002607e48, 0xc002607dc8, 0xc002607dd8?, 0x0, 0xc000f89c00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002607d80 sp=0xc002607c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139c280, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002607e78 sp=0xc002607d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0005730a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002607f30 sp=0xc002607e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002607fe0 sp=0xc002607f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002607fe8 sp=0xc002607fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 303 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 278 [select]: +runtime.gopark(0xc00124c890?, 0x2?, 0x98?, 0xc7?, 0xc00124c83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00124c6a8 sp=0xc00124c688 pc=0x4777ae +runtime.selectgo(0xc00124c890, 0xc00124c838, 0x7f429a422b00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00124c7f8 sp=0xc00124c6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00010ff40, {0xc001150460, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc00124c8c0 sp=0xc00124c7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00010ff40, {0xc001150460, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc00124c958 sp=0xc00124c8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001150120, {0xc001150460, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc00124c9a0 sp=0xc00124c958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001150120}, {0xc001150460, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00124c9e8 sp=0xc00124c9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00110ed80, {0xc001150460, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc00124ca58 sp=0xc00124c9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001150450, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc00124cb28 sp=0xc00124ca58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001160180?, 0xc00110ed80, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc00124cc18 sp=0xc00124cb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001150450, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001152500}, 0xc000c70240?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc00124ccd0 sp=0xc00124cc18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000c074a0, {0x43dcf60?, 0xc001152500}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc00124cf08 sp=0xc00124ccd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0011505a0, {0x43dcf60, 0xc001152500}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc00124cff0 sp=0xc00124cf08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0009d25c0, {0x43dcf60, 0xc001152500}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc00124d028 sp=0xc00124cff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00114c520) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc00124d068 sp=0xc00124d028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00114c520?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc00124d0a8 sp=0xc00124d068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00114c530) + :1 +0x46 fp=0xc00124d0e0 sp=0xc00124d0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c703c0, {0x5b51368, 0xc00114c530}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc00124d320 sp=0xc00124d0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000c6e5e0, {0x5b51038, 0xc00114c520}) + :1 +0xaf fp=0xc00124d370 sp=0xc00124d320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000c6e5e0}, {0x5b4a6f0?, 0xc0009d25c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc00124d3b0 sp=0xc00124d370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc0011505a0}, 0xc000dc3020?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc00124d400 sp=0xc00124d3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc0011505a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc00124d490 sp=0xc00124d400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc000c074a0}, 0xc000dc3020, 0xc00112b580) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc00124d7d0 sp=0xc00124d490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc000c074a0}, 0x3f252a0?, 0xc00114c450?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc00124d848 sp=0xc00124d7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00110ed80, 0xc000c70450, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc00124ddb0 sp=0xc00124d848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00110ed80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc00124df28 sp=0xc00124ddb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00124dfe0 sp=0xc00124df28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00124dfe8 sp=0xc00124dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 264 [select]: +runtime.gopark(0xc00119ffb0?, 0x2?, 0xb4?, 0x1b?, 0xc00119ff74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00119fde8 sp=0xc00119fdc8 pc=0x4777ae +runtime.selectgo(0xc00119ffb0, 0xc00119ff70, 0x0?, 0x0, 0xc000000000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00119ff38 sp=0xc00119fde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc00119ffe0 sp=0xc00119ff38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00119ffe8 sp=0xc00119ffe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 265 [select]: +runtime.gopark(0xc0011d8fb0?, 0x2?, 0xb0?, 0x8d?, 0xc0011d8ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011d8d58 sp=0xc0011d8d38 pc=0x4777ae +runtime.selectgo(0xc0011d8fb0, 0xc0011d8ee0, 0x4aead9?, 0x0, 0xc000f1df30?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011d8ea8 sp=0xc0011d8d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0011d8fe0 sp=0xc0011d8ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011d8fe8 sp=0xc0011d8fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 266 [chan receive]: +runtime.gopark(0x900000000000000?, 0x0?, 0x4?, 0x0?, 0x1?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001192d60 sp=0xc001192d40 pc=0x4777ae +runtime.chanrecv(0xc000f26cc0, 0xc001192e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001192dd8 sp=0xc001192d60 pc=0x441325 +runtime.chanrecv1(0xd?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001192e00 sp=0xc001192dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001192fe0 sp=0xc001192e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001192fe8 sp=0xc001192fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 255 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc000c9df80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001240e20 sp=0xc001240e00 pc=0x4777ae +runtime.chanrecv(0xc000c9df80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001240e98 sp=0xc001240e20 pc=0x441325 +runtime.chanrecv1(0xc000eb9470?, 0x2106cfa?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001240ec0 sp=0xc001240e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc001240fe0 sp=0xc001240ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001240fe8 sp=0xc001240fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 258 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 256 [select]: +runtime.gopark(0xc000a6daa8?, 0x2?, 0x67?, 0x6a?, 0xc000a6d9c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a6d838 sp=0xc000a6d818 pc=0x4777ae +runtime.selectgo(0xc000a6daa8, 0xc000a6d9c0, 0xc000a6d9d0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a6d988 sp=0xc000a6d838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc00098d860) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc000a6dad8 sp=0xc000a6d988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc000f8b560, {0x5b426a8, 0xc000eb9470}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc000a6dcf8 sp=0xc000a6dad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc000a6dfe0 sp=0xc000a6dcf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a6dfe8 sp=0xc000a6dfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 258 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 348 [select]: +runtime.gopark(0xc00129a890?, 0x2?, 0x98?, 0xa7?, 0xc00129a83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00129a6a8 sp=0xc00129a688 pc=0x4777ae +runtime.selectgo(0xc00129a890, 0xc00129a838, 0x7f429a426a00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00129a7f8 sp=0xc00129a6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0007c1a90, {0xc001266b50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc00129a8c0 sp=0xc00129a7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0007c1a90, {0xc001266b50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc00129a958 sp=0xc00129a8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001266ab0, {0xc001266b50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc00129a9a0 sp=0xc00129a958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001266ab0}, {0xc001266b50, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00129a9e8 sp=0xc00129a9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00123cc60, {0xc001266b50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc00129aa58 sp=0xc00129a9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001266b40, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc00129ab28 sp=0xc00129aa58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc00141c140?, 0xc00123cc60, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc00129ac18 sp=0xc00129ab28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001266b40, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0007c1db0}, 0xc000c70240?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc00129acd0 sp=0xc00129ac18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0012340f0, {0x43dcf60?, 0xc0007c1db0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc00129af08 sp=0xc00129acd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001266c90, {0x43dcf60, 0xc0007c1db0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc00129aff0 sp=0xc00129af08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc00127c140, {0x43dcf60, 0xc0007c1db0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc00129b028 sp=0xc00129aff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc000fe1a60) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc00129b068 sp=0xc00129b028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc000fe1a60?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc00129b0a8 sp=0xc00129b068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc000fe1a70) + :1 +0x46 fp=0xc00129b0e0 sp=0xc00129b0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c703c0, {0x5b51368, 0xc000fe1a70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc00129b320 sp=0xc00129b0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000c6e5e0, {0x5b51038, 0xc000fe1a60}) + :1 +0xaf fp=0xc00129b370 sp=0xc00129b320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000c6e5e0}, {0x5b4a6f0?, 0xc00127c140}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc00129b3b0 sp=0xc00129b370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc001266c90}, 0xc000f1fd88?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc00129b400 sp=0xc00129b3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc001266c90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc00129b490 sp=0xc00129b400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0012340f0}, 0xc000f1fd88, 0xc001237540) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc00129b7d0 sp=0xc00129b490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0012340f0}, 0x3f252a0?, 0xc000fe1990?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc00129b848 sp=0xc00129b7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00123cc60, 0xc000c70450, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc00129bdb0 sp=0xc00129b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc00123cc60, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc00129bf28 sp=0xc00129bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00129bfe0 sp=0xc00129bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00129bfe8 sp=0xc00129bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 287 [select]: +runtime.gopark(0xc00115cf70?, 0x4?, 0x1d?, 0x0?, 0xc00115cdd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00115cc00 sp=0xc00115cbe0 pc=0x4777ae +runtime.selectgo(0xc00115cf70, 0xc00115cdd0, 0x1?, 0x0, 0x5b5d9d8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00115cd50 sp=0xc00115cc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001174340) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc00115cfc0 sp=0xc00115cd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc00115cfe0 sp=0xc00115cfc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00115cfe8 sp=0xc00115cfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 304 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 342 [IO wait]: +runtime.gopark(0xc00113efe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00113ef98 sp=0xc00113ef78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00113efd0 sp=0xc00113ef98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42416779d8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00113eff0 sp=0xc00113efd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0010e00a0, 0xc00108a000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00113f038 sp=0xc00113eff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0010e0080, {0xc00108a000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00113f128 sp=0xc00113f038 pc=0x50e5a5 +net.(*netFD).Read(0xc0010e0080, {0xc00108a000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00113f188 sp=0xc00113f128 pc=0x89342b +net.(*conn).Read(0xc00007de18, {0xc00108a000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00113f210 sp=0xc00113f188 pc=0x8b176d +net.(*UnixConn).Read(0xc00007de18, {0xc00108a000, 0x4000, 0x4000}) + :1 +0x54 fp=0xc00113f258 sp=0xc00113f210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0010d9590, {0xc00108a000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc00113f2b8 sp=0xc00113f258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0010b89a8, {0x5b0f6c0, 0xc0010d9590}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc00113f320 sp=0xc00113f2b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc0010b8700, {0x5b0f840?, 0xc00007de18}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc00113f390 sp=0xc00113f320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc0010b8700, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc00113f9c0 sp=0xc00113f390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc0010b8700, {0xc001400000, 0x8000, 0xc001ed4410?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc00113faa8 sp=0xc00113f9c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc000ee7fa0, {0xc001400000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc00113faf0 sp=0xc00113faa8 pc=0x10969ed +bufio.(*Reader).Read(0xc00127a3c0, {0xc000f8e3c0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00113fbb0 sp=0xc00113faf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00127a3c0}, {0xc000f8e3c0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00113fbf8 sp=0xc00113fbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000f8e3c0, 0x9, 0x9}, {0x5b0f0a0, 0xc00127a3c0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00113fc98 sp=0xc00113fbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000f8e380) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00113fdd0 sp=0xc00113fc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000f95440, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc00113ffb0 sp=0xc00113fdd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc00113ffe0 sp=0xc00113ffb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00113ffe8 sp=0xc00113ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 323 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 512 [select]: +runtime.gopark(0xc000a4ffb0?, 0x2?, 0xb4?, 0x1b?, 0xc000a4ff74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a4fde8 sp=0xc000a4fdc8 pc=0x4777ae +runtime.selectgo(0xc000a4ffb0, 0xc000a4ff70, 0x1000000?, 0x0, 0x116e9f5?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a4ff38 sp=0xc000a4fde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc000a4ffe0 sp=0xc000a4ff38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a4ffe8 sp=0xc000a4ffe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 566 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 257 [select]: +runtime.gopark(0xc0011813f8?, 0x2?, 0xc0?, 0x72?, 0xc00118137c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011811e8 sp=0xc0011811c8 pc=0x4777ae +runtime.selectgo(0xc0011813f8, 0xc001181378, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001181338 sp=0xc0011811e8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000830e60, {0xc000eb9540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc001181428 sp=0xc001181338 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000830e60, {0xc000eb9540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc0011814c0 sp=0xc001181428 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000eb9500, {0xc000eb9540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001181508 sp=0xc0011814c0 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000eb9500}, {0xc000eb9540, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001181550 sp=0xc001181508 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001196120, {0xc000eb9540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0011815c0 sp=0xc001181550 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000eb9530, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001181690 sp=0xc0011815c0 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001196120, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001181780 sp=0xc001181690 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000eb9530, {0x7f424347ae80, 0x779c980}, 0x7f429a503290?, {0x0, 0x0}, {0x43f1c20, 0xc0005ba0e0}, 0x456976?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001181838 sp=0xc001181780 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc000eb7110, {0x43f1c20?, 0xc0005ba0e0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc001181a98 sp=0xc001181838 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00094ff14?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001181af8 sp=0xc001181a98 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00094fe60, 0xc001181cb0, 0xc001181c80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc001181ba0 sp=0xc001181af8 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00094fe60, {0x43f1c20, 0xc0005ba0e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001181ce0 sp=0xc001181ba0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc000ed13c0, {0x43f1c20, 0xc0005ba0e0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc001181ea8 sp=0xc001181ce0 pc=0x12ac5b9 +github.com/rancher/opni/pkg/plugins/apis/apiextensions.(*streamAPIExtensionConnectInternalClient).Recv(0xc000e9cbf0) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go:249 +0x66 fp=0xc001181ee8 sp=0xc001181ea8 pc=0x218dd66 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc001181fe0 sp=0xc001181ee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001181fe8 sp=0xc001181fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 256 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 372 [select]: +runtime.gopark(0xc00125c890?, 0x2?, 0x4e?, 0x1a?, 0xc00125c83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00125c6a8 sp=0xc00125c688 pc=0x4777ae +runtime.selectgo(0xc00125c890, 0xc00125c838, 0x7f429a428b00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00125c7f8 sp=0xc00125c6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00139c4b0, {0xc001394fa0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc00125c8c0 sp=0xc00125c7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00139c4b0, {0xc001394fa0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc00125c958 sp=0xc00125c8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001394d50, {0xc001394fa0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc00125c9a0 sp=0xc00125c958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001394d50}, {0xc001394fa0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00125c9e8 sp=0xc00125c9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00110f440, {0xc001394fa0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc00125ca58 sp=0xc00125c9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001394f90, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc00125cb28 sp=0xc00125ca58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0014802c0?, 0xc00110f440, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc00125cc18 sp=0xc00125cb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001394f90, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc00139c8c0}, 0xc000c4a360?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc00125ccd0 sp=0xc00125cc18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000c07680, {0x43dcf60?, 0xc00139c8c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc00125cf08 sp=0xc00125ccd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0013950e0, {0x43dcf60, 0xc00139c8c0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc00125cff0 sp=0xc00125cf08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0009d3e00, {0x43dcf60, 0xc00139c8c0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc00125d028 sp=0xc00125cff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00114dd40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc00125d068 sp=0xc00125d028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00114dd40?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc00125d0a8 sp=0xc00125d068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00114dd50) + :1 +0x46 fp=0xc00125d0e0 sp=0xc00125d0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c4a4e0, {0x5b51368, 0xc00114dd50}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc00125d320 sp=0xc00125d0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000bfd440, {0x5b51038, 0xc00114dd40}) + :1 +0xaf fp=0xc00125d370 sp=0xc00125d320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000bfd440}, {0x5b4a6f0?, 0xc0009d3e00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc00125d3b0 sp=0xc00125d370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc0013950e0}, 0xc000dc3c08?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc00125d400 sp=0xc00125d3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc0013950e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc00125d490 sp=0xc00125d400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000c07680}, 0xc000dc3c08, 0xc0011618c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc00125d7d0 sp=0xc00125d490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000c07680}, 0x3f252a0?, 0xc00114dc70?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc00125d848 sp=0xc00125d7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc00110f440, 0xc000c4a570, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc00125ddb0 sp=0xc00125d848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc00110f440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc00125df28 sp=0xc00125ddb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00125dfe0 sp=0xc00125df28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00125dfe8 sp=0xc00125dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 291 [select]: +runtime.gopark(0xc001099fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001099f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001099de8 sp=0xc001099dc8 pc=0x4777ae +runtime.selectgo(0xc001099fb0, 0xc001099f70, 0x0?, 0x0, 0xc000000000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001099f38 sp=0xc001099de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001099fe0 sp=0xc001099f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001099fe8 sp=0xc001099fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 292 [select]: +runtime.gopark(0xc001245fb0?, 0x2?, 0xb0?, 0x5d?, 0xc001245ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001245d58 sp=0xc001245d38 pc=0x4777ae +runtime.selectgo(0xc001245fb0, 0xc001245ee0, 0x0?, 0x0, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001245ea8 sp=0xc001245d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001245fe0 sp=0xc001245ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001245fe8 sp=0xc001245fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 293 [chan receive]: +runtime.gopark(0x7700000000000000?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001191d60 sp=0xc001191d40 pc=0x4777ae +runtime.chanrecv(0xc00123e120, 0xc001191e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001191dd8 sp=0xc001191d60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001191e00 sp=0xc001191dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001191fe0 sp=0xc001191e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001191fe8 sp=0xc001191fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 306 [select]: +runtime.gopark(0xc00113b710?, 0x2?, 0x0?, 0x0?, 0xc00113b6bc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00113b528 sp=0xc00113b508 pc=0x4777ae +runtime.selectgo(0xc00113b710, 0xc00113b6b8, 0x7f429a41c200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00113b678 sp=0xc00113b528 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000831400, {0xc000eb98a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc00113b740 sp=0xc00113b678 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000831400, {0xc000eb98a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc00113b7d8 sp=0xc00113b740 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000eb9800, {0xc000eb98a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc00113b820 sp=0xc00113b7d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc000eb9800}, {0xc000eb98a0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00113b868 sp=0xc00113b820 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001196360, {0xc000eb98a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc00113b8d8 sp=0xc00113b868 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc000eb9890, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc00113b9a8 sp=0xc00113b8d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001196360, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc00113ba98 sp=0xc00113b9a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc000eb9890, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc000905b90}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc00113bb50 sp=0xc00113ba98 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000e842d0, {0x43f1c20?, 0xc000905b90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc00113bd88 sp=0xc00113bb50 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc000eb99e0, {0x43f1c20, 0xc000905b90}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc00113be70 sp=0xc00113bd88 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0005fb7c0, {0x43f1c20, 0xc000905b90}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc00113bea8 sp=0xc00113be70 pc=0x1fd94b2 +github.com/rancher/opni/pkg/plugins/apis/apiextensions.(*streamAPIExtensionConnectInternalServer).Recv(0xc000e9ce00) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go:303 +0x66 fp=0xc00113bee8 sp=0xc00113bea8 pc=0x218e046 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc00113bfe0 sp=0xc00113bee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00113bfe8 sp=0xc00113bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 252 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 294 [select]: +runtime.gopark(0xc001246f20?, 0x2?, 0xb4?, 0x1b?, 0xc001246eb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001246d28 sp=0xc001246d08 pc=0x4777ae +runtime.selectgo(0xc001246f20, 0xc001246eb0, 0x0?, 0x0, 0xc001246f18?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001246e78 sp=0xc001246d28 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc00110a780, 0xc000fe0ed0, 0xc000fe0ee0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001246f50 sp=0xc001246e78 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore(0xc00110a780, {0x5b43370?, 0xc000fe0dd0}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:214 +0x1a8 fp=0xc001246f98 sp=0xc001246f50 pc=0x21a84a8 +github.com/rancher/opni/pkg/gateway.NewGateway.func2.1() + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x62 fp=0xc001246fe0 sp=0xc001246f98 pc=0x22b18c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001246fe8 sp=0xc001246fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.NewGateway.func2 in goroutine 233 + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x3cc + +goroutine 270 [select]: +runtime.gopark(0xc00109df70?, 0x2?, 0xb4?, 0x1b?, 0xc00109debc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00109dd20 sp=0xc00109dd00 pc=0x4777ae +runtime.selectgo(0xc00109df70, 0xc00109deb8, 0x4aead9?, 0x0, 0x215774d?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00109de70 sp=0xc00109dd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00006d580, {0x5b426e0, 0xc000b68b40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc00109dfa0 sp=0xc00109de70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc00109dfe0 sp=0xc00109dfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00109dfe8 sp=0xc00109dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 262 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 271 [select]: +runtime.gopark(0xc001213f70?, 0x2?, 0xb4?, 0x1b?, 0xc001213ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001213d20 sp=0xc001213d00 pc=0x4777ae +runtime.selectgo(0xc001213f70, 0xc001213eb8, 0x472ddb?, 0x0, 0x4ad421?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001213e70 sp=0xc001213d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00006d6e0, {0x5b426e0, 0xc000b68b90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001213fa0 sp=0xc001213e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001213fe0 sp=0xc001213fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001213fe8 sp=0xc001213fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 262 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 297 [select]: +runtime.gopark(0xc001251448?, 0x2?, 0xd0?, 0x12?, 0xc0012513fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001251278 sp=0xc001251258 pc=0x4777ae +runtime.selectgo(0xc001251448, 0xc0012513f8, 0xc000b6efe8?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012513c8 sp=0xc001251278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00123c7e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001251478 sp=0xc0012513c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00123a410, {0x42219c0?, 0xc000ff9620}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0012516d8 sp=0xc001251478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00123c654?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001251738 sp=0xc0012516d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00123c5a0, 0xc0012518f0, 0xc0012518c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc0012517e0 sp=0xc001251738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00123c5a0, {0x42219c0, 0xc000ff9620}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001251920 sp=0xc0012517e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc000ff96e0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc000ff95f0}, {0x42219c0, 0xc000ff9620}, 0x0?, {0xc0005fac80, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc0012519c0 sp=0xc001251920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc000bdcaa0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc000ff95f0}, {0x42219c0, 0xc000ff9620}, 0xc0009c7400, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc001251d30 sp=0xc0012519c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc0009c7400, {0x5b426e0, 0xc000bdcaa0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc000ff95f0}, {0x42219c0, 0xc000ff9620}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc001251e58 sp=0xc001251d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseKeyValueStore(0xc000c6f160, {0x5b426e0, 0xc000bdcaa0}, 0x7769700?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:69 +0xee fp=0xc001251ee8 sp=0xc001251e58 pc=0x21ab96e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore.func2(0x3) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:219 +0xf2 fp=0xc001251f58 sp=0xc001251ee8 pc=0x21a85d2 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc001251fe0 sp=0xc001251f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001251fe8 sp=0xc001251fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 294 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 272 [select]: +runtime.gopark(0xc001215f70?, 0x2?, 0xb4?, 0x1b?, 0xc001215ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001215d20 sp=0xc001215d00 pc=0x4777ae +runtime.selectgo(0xc001215f70, 0xc001215eb8, 0x472ddb?, 0x0, 0x4ad421?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001215e70 sp=0xc001215d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc00006dac0, {0x5b426e0, 0xc000b68d70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001215fa0 sp=0xc001215e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001215fe0 sp=0xc001215fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001215fe8 sp=0xc001215fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 177 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 273 [select]: +runtime.gopark(0xc000a6bf70?, 0x2?, 0xb4?, 0x1b?, 0xc000a6bebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a6bd20 sp=0xc000a6bd00 pc=0x4777ae +runtime.selectgo(0xc000a6bf70, 0xc000a6beb8, 0x4b3745?, 0x0, 0xc000bdcb08?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a6be70 sp=0xc000a6bd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b30040, {0x5b426e0, 0xc000b68dc0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc000a6bfa0 sp=0xc000a6be70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc000a6bfe0 sp=0xc000a6bfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a6bfe8 sp=0xc000a6bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 177 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 298 [chan receive]: +runtime.gopark(0x78?, 0x7769700?, 0x70?, 0x67?, 0xc000bdcb08?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001158ef8 sp=0xc001158ed8 pc=0x4777ae +runtime.chanrecv(0xc000c9ccc0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001158f70 sp=0xc001158ef8 pc=0x441325 +runtime.chanrecv1(0xc000bdcaa0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001158f98 sp=0xc001158f70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001158fe0 sp=0xc001158f98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001158fe8 sp=0xc001158fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 296 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 299 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001219978 sp=0xc001219958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0012199b0 sp=0xc001219978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677ad0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0012199d0 sp=0xc0012199b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d175a0, 0xc001219a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001219a18 sp=0xc0012199d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000d17580) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001219b08 sp=0xc001219a18 pc=0x513f65 +net.(*netFD).accept(0xc000d17580) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001219c00 sp=0xc001219b08 pc=0x8967be +net.(*UnixListener).accept(0xc000ff98c0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001219c40 sp=0xc001219c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc000ff98c0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001219cb0 sp=0xc001219c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc000ee7a00) + :1 +0x44 fp=0xc001219ce8 sp=0xc001219cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc0004efe00, {0x5b2c6a0, 0xc000ee7a00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001219f18 sp=0xc001219ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc001219f68 sp=0xc001219f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc000ee7b60?, 0xc000fe10e0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001219fa8 sp=0xc001219f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001219fe0 sp=0xc001219fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001219fe8 sp=0xc001219fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 296 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 300 [select]: +runtime.gopark(0xc001247f38?, 0x2?, 0x0?, 0x0?, 0xc001247f14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001247d90 sp=0xc001247d70 pc=0x4777ae +runtime.selectgo(0xc001247f38, 0xc001247f10, 0xc001247f18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001247ee0 sp=0xc001247d90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc001247f68 sp=0xc001247ee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc000f1fa28?, 0xc000fe1100?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001247fa8 sp=0xc001247f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001247fe0 sp=0xc001247fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001247fe8 sp=0xc001247fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 296 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 307 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x10?, 0x8e?, 0x10?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001258cf8 sp=0xc001258cd8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001258d70 sp=0xc001258cf8 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0x5b55580?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001258d98 sp=0xc001258d70 pc=0x440f72 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseKeyValueStore(0xc0005a8c80, {0x5b43f90?, 0xc001490050}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:185 +0xc25 fp=0xc001258fd0 sp=0xc001258d98 pc=0x3970905 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseKeyValueStore(0xc00091c300, {0x4b05a9?, 0xc00113d0f0?}, 0xc0011ec720) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:123 +0x18d fp=0xc001259098 sp=0xc001258fd0 pc=0x21a6c2d +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler.func1({0x5b426a8, 0xc0011ec7e0}, {0x4278de0?, 0xc0011ec720}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:187 +0x83 fp=0xc0012590f0 sp=0xc001259098 pc=0x21ac823 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc0011ec7e0}, {0x4278de0, 0xc0011ec720}, 0xc00043b060, 0xc000a736b0) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001259148 sp=0xc0012590f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc0011ec7e0}, {0x4278de0, 0xc0011ec720}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc0012591e8 sp=0xc001259148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc0011ec6f0}, {0x4278de0, 0xc0011ec720}, 0xc00043b060, 0xc0011ca740) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc0012596b8 sp=0xc0012591e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc0011ec6f0}, {0x4278de0, 0xc0011ec720}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001259740 sp=0xc0012596b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler({0x4278ea0?, 0xc00091c300}, {0x5b426a8, 0xc0011ec6f0}, 0xc000905810, 0xc000b75440) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:189 +0x1c4 fp=0xc0012597d0 sp=0xc001259740 pc=0x21ac684 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc001197200, 0xc000c70f60, 0x771ecd0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001259db0 sp=0xc0012597d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc001197200, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001259f28 sp=0xc001259db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001259fe0 sp=0xc001259f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001259fe8 sp=0xc001259fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1293 [select]: +runtime.gopark(0xc001ef6f48?, 0x2?, 0x10?, 0x98?, 0xc001ef6f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef6d80 sp=0xc001ef6d60 pc=0x4777ae +runtime.selectgo(0xc001ef6f48, 0xc001ef6f00, 0x1?, 0x0, 0x86?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ef6ed0 sp=0xc001ef6d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).Recv(0xc001ff19c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:234 +0xeb fp=0xc001ef6f78 sp=0xc001ef6ed0 pc=0x2155d2b +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc0026a4a00) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc001ef6fc0 sp=0xc001ef6f78 pc=0x215759f +github.com/hashicorp/go-plugin.newGRPCClient.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x34 fp=0xc001ef6fe0 sp=0xc001ef6fc0 pc=0x21599b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef6fe8 sp=0xc001ef6fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 1361 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x5b7 + +goroutine 457 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xa0?, 0x2f?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00157fcd0 sp=0xc00157fcb0 pc=0x4777ae +runtime.chanrecv(0xc000f1acc0, 0xc00157fde0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00157fd48 sp=0xc00157fcd0 pc=0x441325 +runtime.chanrecv1(0xc00157fdf0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00157fd70 sp=0xc00157fd48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc00157ff30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc00157fe00 sp=0xc00157fd70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc000ee4410, 0x420c60?, 0xc000ff8a20) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc00157ffa0 sp=0xc00157fe00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc00157ffe0 sp=0xc00157ffa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00157ffe8 sp=0xc00157ffe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 456 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 308 [select]: +runtime.gopark(0xc0010f2890?, 0x2?, 0x98?, 0x27?, 0xc0010f283c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010f26a8 sp=0xc0010f2688 pc=0x4777ae +runtime.selectgo(0xc0010f2890, 0xc0010f2838, 0x7f429a420700?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010f27f8 sp=0xc0010f26a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0011f2190, {0xc0011ecc40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0010f28c0 sp=0xc0010f27f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0011f2190, {0xc0011ecc40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0010f2958 sp=0xc0010f28c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0011ec9c0, {0xc0011ecc40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0010f29a0 sp=0xc0010f2958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0011ec9c0}, {0xc0011ecc40, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0010f29e8 sp=0xc0010f29a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001197320, {0xc0011ecc40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0010f2a58 sp=0xc0010f29e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0011ecc30, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0010f2b28 sp=0xc0010f2a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0011cb380?, 0xc001197320, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0010f2c18 sp=0xc0010f2b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0011ecc30, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0011f25a0}, 0xc000c4a360?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0010f2cd0 sp=0xc0010f2c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000e84f00, {0x43dcf60?, 0xc0011f25a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0010f2f08 sp=0xc0010f2cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0011ecd80, {0x43dcf60, 0xc0011f25a0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0010f2ff0 sp=0xc0010f2f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc00043b340, {0x43dcf60, 0xc0011f25a0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0010f3028 sp=0xc0010f2ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc0011cec70) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0010f3068 sp=0xc0010f3028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc0011cec70?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0010f30a8 sp=0xc0010f3068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc0011cec80) + :1 +0x46 fp=0xc0010f30e0 sp=0xc0010f30a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c4a4e0, {0x5b51368, 0xc0011cec80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0010f3320 sp=0xc0010f30e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000bfd440, {0x5b51038, 0xc0011cec70}) + :1 +0xaf fp=0xc0010f3370 sp=0xc0010f3320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000bfd440}, {0x5b4a6f0?, 0xc00043b340}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0010f33b0 sp=0xc0010f3370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc0011ecd80}, 0xc000a738c0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0010f3400 sp=0xc0010f33b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc0011ecd80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0010f3490 sp=0xc0010f3400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000e84f00}, 0xc000a738c0, 0xc0011ca980) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0010f37d0 sp=0xc0010f3490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc000e84f00}, 0x3f252a0?, 0xc0011ceba0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0010f3848 sp=0xc0010f37d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc001197320, 0xc000c4a570, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0010f3db0 sp=0xc0010f3848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc001197320, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0010f3f28 sp=0xc0010f3db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0010f3fe0 sp=0xc0010f3f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010f3fe8 sp=0xc0010f3fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 559 [select]: +runtime.gopark(0xc000f77e38?, 0x2?, 0xb4?, 0x1b?, 0xc000f77c5c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b9a58 sp=0xc0029b9a38 pc=0x4777ae +runtime.selectgo(0xc0029b9e38, 0xc000f77c58, 0x0?, 0x0, 0x4113100?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029b9ba8 sp=0xc0029b9a58 pc=0x48916b +github.com/rancher/opni/pkg/gateway.(*GatewayGRPCServer).ListenAndServe(0xc000c95da0, {0x5b426e0, 0xc0020885a0}) + /home/ec2-user/opni/pkg/gateway/grpc.go:107 +0x106f fp=0xc0029b9e68 sp=0xc0029b9ba8 pc=0x22b4b0f +github.com/rancher/opni/pkg/gateway.(*Gateway).ListenAndServe.func2() + /home/ec2-user/opni/pkg/gateway/gateway.go:338 +0x86 fp=0xc0029b9fa0 sp=0xc0029b9e68 pc=0x22b2686 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0029b9fe0 sp=0xc0029b9fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b9fe8 sp=0xc0029b9fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 581 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 428 [chan receive]: +runtime.gopark(0xc000b3d020?, 0x27?, 0x0?, 0x0?, 0x460fed5?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00148bef8 sp=0xc00148bed8 pc=0x4777ae +runtime.chanrecv(0xc0010886c0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00148bf70 sp=0xc00148bef8 pc=0x441325 +runtime.chanrecv1(0xc000762460?, 0x47e975?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00148bf98 sp=0xc00148bf70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc00148bfe0 sp=0xc00148bf98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00148bfe8 sp=0xc00148bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 424 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 343 [select]: +runtime.gopark(0xc002499e70?, 0x2?, 0x0?, 0x0?, 0xc002499df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002499c58 sp=0xc002499c38 pc=0x4777ae +runtime.selectgo(0xc002499e70, 0xc002499df0, 0xc002499e00?, 0x0, 0xc0010b8700?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002499da8 sp=0xc002499c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0007c17c0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002499ea0 sp=0xc002499da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006109a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002499f58 sp=0xc002499ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc002499fe0 sp=0xc002499f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002499fe8 sp=0xc002499fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 323 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 351 [select]: +runtime.gopark(0xc0010f6890?, 0x2?, 0x98?, 0x67?, 0xc0010f683c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010f66a8 sp=0xc0010f6688 pc=0x4777ae +runtime.selectgo(0xc0010f6890, 0xc0010f6838, 0x7f429a427900?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010f67f8 sp=0xc0010f66a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001434190, {0xc001472d30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0010f68c0 sp=0xc0010f67f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001434190, {0xc001472d30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0010f6958 sp=0xc0010f68c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0012672f0, {0xc001472d30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0010f69a0 sp=0xc0010f6958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0012672f0}, {0xc001472d30, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0010f69e8 sp=0xc0010f69a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00123d200, {0xc001472d30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0010f6a58 sp=0xc0010f69e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001472d20, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0010f6b28 sp=0xc0010f6a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc00141d440?, 0xc00123d200, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0010f6c18 sp=0xc0010f6b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001472d20, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001434640}, 0xc000c4a360?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0010f6cd0 sp=0xc0010f6c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0012341e0, {0x43dcf60?, 0xc001434640}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0010f6f08 sp=0xc0010f6cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001472e70, {0x43dcf60, 0xc001434640}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0010f6ff0 sp=0xc0010f6f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc00127d2a0, {0x43dcf60, 0xc001434640}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0010f7028 sp=0xc0010f6ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00158a120) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0010f7068 sp=0xc0010f7028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00158a120?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0010f70a8 sp=0xc0010f7068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00158a130) + :1 +0x46 fp=0xc0010f70e0 sp=0xc0010f70a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c4a4e0, {0x5b51368, 0xc00158a130}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0010f7320 sp=0xc0010f70e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000bfd440, {0x5b51038, 0xc00158a120}) + :1 +0xaf fp=0xc0010f7370 sp=0xc0010f7320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000bfd440}, {0x5b4a6f0?, 0xc00127d2a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0010f73b0 sp=0xc0010f7370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc001472e70}, 0xc001422db0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0010f7400 sp=0xc0010f73b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc001472e70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0010f7490 sp=0xc0010f7400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc0012341e0}, 0xc001422db0, 0xc00141ca40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0010f77d0 sp=0xc0010f7490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc0012341e0}, 0x3f252a0?, 0xc00158a050?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0010f7848 sp=0xc0010f77d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc00123d200, 0xc000c4a570, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0010f7db0 sp=0xc0010f7848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc00123d200, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0010f7f28 sp=0xc0010f7db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0010f7fe0 sp=0xc0010f7f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010f7fe8 sp=0xc0010f7fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 284 [IO wait]: +runtime.gopark(0xc0010ecfe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010ecf98 sp=0xc0010ecf78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0010ecfd0 sp=0xc0010ecf98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42416777e8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0010ecff0 sp=0xc0010ecfd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0010e02a0, 0xc00139e600?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0010ed038 sp=0xc0010ecff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0010e0280, {0xc00139e600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0010ed128 sp=0xc0010ed038 pc=0x50e5a5 +net.(*netFD).Read(0xc0010e0280, {0xc00139e600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0010ed188 sp=0xc0010ed128 pc=0x89342b +net.(*conn).Read(0xc00007de20, {0xc00139e600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0010ed210 sp=0xc0010ed188 pc=0x8b176d +net.(*UnixConn).Read(0xc00007de20, {0xc00139e600, 0x600, 0x600}) + :1 +0x54 fp=0xc0010ed258 sp=0xc0010ed210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc000f1ffe0, {0xc00139e600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0010ed2b8 sp=0xc0010ed258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0010b8d28, {0x5b0f6c0, 0xc000f1ffe0}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0010ed320 sp=0xc0010ed2b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc0010b8a80, {0x5b0f840?, 0xc00007de20}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0010ed390 sp=0xc0010ed320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc0010b8a80, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0010ed9c0 sp=0xc0010ed390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc0010b8a80, {0xc0013a6000, 0x8000, 0x4?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0010edaa8 sp=0xc0010ed9c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0009d3c40, {0xc0013a6000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0010edaf0 sp=0xc0010edaa8 pc=0x10969ed +bufio.(*Reader).Read(0xc0013993e0, {0xc000c49620, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0010edbb0 sp=0xc0010edaf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0013993e0}, {0xc000c49620, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0010edbf8 sp=0xc0010edbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000c49620, 0x9, 0x9}, {0x5b0f0a0, 0xc0013993e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0010edc98 sp=0xc0010edbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000c495e0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0010eddd0 sp=0xc0010edc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001128240, 0xc000c46e50?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0010edfb0 sp=0xc0010eddd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0010edfe0 sp=0xc0010edfb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010edfe8 sp=0xc0010edfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 322 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 558 [select]: +runtime.gopark(0xc0017dd140?, 0x3?, 0xa0?, 0xea?, 0xc00209dd9a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029c57d0 sp=0xc0029c57b0 pc=0x4777ae +runtime.selectgo(0xc0017dd140, 0xc00209dd94, 0x0?, 0x0, 0xc001bf1ad0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029c5920 sp=0xc0029c57d0 pc=0x48916b +reflect.rselect({0xc0029c5b08, 0x3, 0x7f4241663488?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:589 +0x345 fp=0xc0029c59d0 sp=0xc0029c5920 pc=0x4a8f05 +reflect.Select({0xc00163bea0, 0x3, 0x2?}) + /home/ec2-user/sdk/go1.21.0/src/reflect/value.go:3104 +0xb8a fp=0xc0029c5b98 sp=0xc0029c59d0 pc=0x57c44a +github.com/rancher/opni/pkg/util.WaitAll({0x5b426e0, 0xc001fee370}, 0xc001bf1a20, {0xc0029c5e08, 0x2, 0x2}) + /home/ec2-user/opni/pkg/util/servers.go:51 +0x3ea fp=0xc0029c5d18 sp=0xc0029c5b98 pc=0x176ae8a +github.com/rancher/opni/pkg/gateway.(*GatewayHTTPServer).ListenAndServe(0xc0008ab5c0, {0x5b426e0, 0xc0020885a0}) + /home/ec2-user/opni/pkg/gateway/http.go:185 +0x5a7 fp=0xc0029c5e68 sp=0xc0029c5d18 pc=0x22b8427 +github.com/rancher/opni/pkg/gateway.(*Gateway).ListenAndServe.func1() + /home/ec2-user/opni/pkg/gateway/gateway.go:327 +0x86 fp=0xc0029c5fa0 sp=0xc0029c5e68 pc=0x22b28a6 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0029c5fe0 sp=0xc0029c5fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029c5fe8 sp=0xc0029c5fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 581 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 283 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011178d8 sp=0xc0011178b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001117910 sp=0xc0011178d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42416775f8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001117930 sp=0xc001117910 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000dbb3a0, 0xc001170001?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001117978 sp=0xc001117930 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000dbb380) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001117a68 sp=0xc001117978 pc=0x513f65 +net.(*netFD).accept(0xc000dbb380) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001117b60 sp=0xc001117a68 pc=0x8967be +net.(*TCPListener).accept(0xc0009d26e0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc001117ba8 sp=0xc001117b60 pc=0x8c383e +net.(*TCPListener).Accept(0xc0009d26e0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc001117c18 sp=0xc001117ba8 pc=0x8c1ba5 +crypto/tls.(*listener).Accept(0xc000dc3440) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/tls.go:66 +0x43 fp=0xc001117c70 sp=0xc001117c18 pc=0x9b2d43 +net/http.(*onceCloseListener).Accept(0xc0011511d0) + :1 +0x44 fp=0xc001117ca8 sp=0xc001117c70 pc=0xab5544 +net/http.(*Server).Serve(0xc000c07590, {0x5b2a5b0, 0xc000dc3440}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc001117ea0 sp=0xc001117ca8 pc=0xa7859f +net/http.Serve(...) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2595 +github.com/gin-gonic/gin.(*Engine).RunListener(0xc000c7c4e0, {0x5b2a5b0, 0xc000dc3440}) + /home/ec2-user/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/gin.go:565 +0x370 fp=0xc001117f90 sp=0xc001117ea0 pc=0x171e870 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure.func1() + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:72 +0x5c fp=0xc001117fe0 sp=0xc001117f90 pc=0x262a57c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001117fe8 sp=0xc001117fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure in goroutine 279 + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:71 +0x26d + +goroutine 311 [select]: +runtime.gopark(0xc0012aafb0?, 0x2?, 0xb4?, 0x1b?, 0xc0012aaf74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012aade8 sp=0xc0012aadc8 pc=0x4777ae +runtime.selectgo(0xc0012aafb0, 0xc0012aaf70, 0x4aead9?, 0x0, 0x4ad421?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012aaf38 sp=0xc0012aade8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0012aafe0 sp=0xc0012aaf38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012aafe8 sp=0xc0012aafe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 312 [select]: +runtime.gopark(0xc001241fb0?, 0x2?, 0xb0?, 0x1d?, 0xc001241ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001241d58 sp=0xc001241d38 pc=0x4777ae +runtime.selectgo(0xc001241fb0, 0xc001241ee0, 0x1?, 0x0, 0x4b0571?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001241ea8 sp=0xc001241d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001241fe0 sp=0xc001241ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001241fe8 sp=0xc001241fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 313 [chan receive]: +runtime.gopark(0x77000000004b0571?, 0x12a7458?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00119cd60 sp=0xc00119cd40 pc=0x4777ae +runtime.chanrecv(0xc00119aa80, 0xc00119ce60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00119cdd8 sp=0xc00119cd60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00119ce00 sp=0xc00119cdd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc00119cfe0 sp=0xc00119ce00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00119cfe8 sp=0xc00119cfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 364 [select]: +runtime.gopark(0xc0011dbfb0?, 0x2?, 0xb4?, 0x1b?, 0xc0011dbf74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011dbde8 sp=0xc0011dbdc8 pc=0x4777ae +runtime.selectgo(0xc0011dbfb0, 0xc0011dbf70, 0xc00128b440?, 0x0, 0xc001286d80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011dbf38 sp=0xc0011dbde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0011dbfe0 sp=0xc0011dbf38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011dbfe8 sp=0xc0011dbfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 329 [select]: +runtime.gopark(0xc001294fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001294f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001294de8 sp=0xc001294dc8 pc=0x4777ae +runtime.selectgo(0xc001294fb0, 0xc001294f70, 0x4aead9?, 0x0, 0x4ad421?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001294f38 sp=0xc001294de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001294fe0 sp=0xc001294f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001294fe8 sp=0xc001294fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 330 [select]: +runtime.gopark(0xc001295fb0?, 0x2?, 0xb0?, 0x5d?, 0xc001295ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001295d58 sp=0xc001295d38 pc=0x4777ae +runtime.selectgo(0xc001295fb0, 0xc001295ee0, 0x448855?, 0x0, 0x400a060?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001295ea8 sp=0xc001295d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001295fe0 sp=0xc001295ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001295fe8 sp=0xc001295fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 331 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001296d60 sp=0xc001296d40 pc=0x4777ae +runtime.chanrecv(0xc001286240, 0xc001296e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001296dd8 sp=0xc001296d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001296e00 sp=0xc001296dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001296fe0 sp=0xc001296e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001296fe8 sp=0xc001296fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 332 [select]: +runtime.gopark(0xc0011fa890?, 0x2?, 0x98?, 0xa7?, 0xc0011fa83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011fa6a8 sp=0xc0011fa688 pc=0x4777ae +runtime.selectgo(0xc0011fa890, 0xc0011fa838, 0x7f429a41c800?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011fa7f8 sp=0xc0011fa6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00128c0a0, {0xc0010e4cd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0011fa8c0 sp=0xc0011fa7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00128c0a0, {0xc0010e4cd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0011fa958 sp=0xc0011fa8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0010e4c30, {0xc0010e4cd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0011fa9a0 sp=0xc0011fa958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0010e4c30}, {0xc0010e4cd0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0011fa9e8 sp=0xc0011fa9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0010dcb40, {0xc0010e4cd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0011faa58 sp=0xc0011fa9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0010e4cc0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0011fab28 sp=0xc0011faa58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001282e00?, 0xc0010dcb40, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0011fac18 sp=0xc0011fab28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0010e4cc0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc00128c3c0}, 0xc000c70240?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0011facd0 sp=0xc0011fac18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0010a6690, {0x43dcf60?, 0xc00128c3c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0011faf08 sp=0xc0011facd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0010e4e10, {0x43dcf60, 0xc00128c3c0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0011faff0 sp=0xc0011faf08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000b30640, {0x43dcf60, 0xc00128c3c0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0011fb028 sp=0xc0011faff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc0010b5f90) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0011fb068 sp=0xc0011fb028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc0010b5f90?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0011fb0a8 sp=0xc0011fb068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc0010b5fa0) + :1 +0x46 fp=0xc0011fb0e0 sp=0xc0011fb0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c703c0, {0x5b51368, 0xc0010b5fa0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0011fb320 sp=0xc0011fb0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000c6e5e0, {0x5b51038, 0xc0010b5f90}) + :1 +0xaf fp=0xc0011fb370 sp=0xc0011fb320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000c6e5e0}, {0x5b4a6f0?, 0xc000b30640}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0011fb3b0 sp=0xc0011fb370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc0010e4e10}, 0xc0010d8588?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0011fb400 sp=0xc0011fb3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc0010e4e10}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0011fb490 sp=0xc0011fb400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0010a6690}, 0xc0010d8588, 0xc001282200) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0011fb7d0 sp=0xc0011fb490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0010a6690}, 0x3f252a0?, 0xc0010b5ec0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0011fb848 sp=0xc0011fb7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc0010dcb40, 0xc000c70450, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0011fbdb0 sp=0xc0011fb848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc0010dcb40, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0011fbf28 sp=0xc0011fbdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0011fbfe0 sp=0xc0011fbf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011fbfe8 sp=0xc0011fbfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 365 [select]: +runtime.gopark(0xc0012b0fb0?, 0x2?, 0xb0?, 0xd?, 0xc0012b0ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012b0d58 sp=0xc0012b0d38 pc=0x4777ae +runtime.selectgo(0xc0012b0fb0, 0xc0012b0ee0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012b0ea8 sp=0xc0012b0d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0012b0fe0 sp=0xc0012b0ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012b0fe8 sp=0xc0012b0fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1156 [select]: +runtime.gopark(0xc002d29f90?, 0x2?, 0x30?, 0x9d?, 0xc002d29e94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d29ce8 sp=0xc002d29cc8 pc=0x4777ae +runtime.selectgo(0xc002d29f90, 0xc002d29e90, 0xc00263dbeb?, 0x0, 0xc001487ea8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d29e38 sp=0xc002d29ce8 pc=0x48916b +github.com/rancher/opni/pkg/storage/jetstream.(*JetStreamStore).WatchClusters.func1() + /home/ec2-user/opni/pkg/storage/jetstream/cluster_store.go:247 +0x18c fp=0xc002d29fe0 sp=0xc002d29e38 pc=0x249e4ac +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d29fe8 sp=0xc002d29fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.(*JetStreamStore).WatchClusters in goroutine 1133 + /home/ec2-user/opni/pkg/storage/jetstream/cluster_store.go:244 +0xbcd + +goroutine 335 [select]: +runtime.gopark(0xc001293fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001293f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001293de8 sp=0xc001293dc8 pc=0x4777ae +runtime.selectgo(0xc001293fb0, 0xc001293f70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001293f38 sp=0xc001293de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001293fe0 sp=0xc001293f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001293fe8 sp=0xc001293fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 134 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 336 [select]: +runtime.gopark(0xc001292fb0?, 0x2?, 0xb0?, 0x2d?, 0xc001292ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001292d58 sp=0xc001292d38 pc=0x4777ae +runtime.selectgo(0xc001292fb0, 0xc001292ee0, 0x448855?, 0x0, 0x400a060?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001292ea8 sp=0xc001292d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001292fe0 sp=0xc001292ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001292fe8 sp=0xc001292fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 337 [chan receive]: +runtime.gopark(0xc0010b5cb0?, 0xc00070f822?, 0xe?, 0x0?, 0x5b058a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001297d60 sp=0xc001297d40 pc=0x4777ae +runtime.chanrecv(0xc0012867e0, 0xc001297e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001297dd8 sp=0xc001297d60 pc=0x441325 +runtime.chanrecv1(0xc0010b5cb0?, 0xc00070f822?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001297e00 sp=0xc001297dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001297fe0 sp=0xc001297e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001297fe8 sp=0xc001297fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 354 [select]: +runtime.gopark(0xc0012a8890?, 0x2?, 0x98?, 0x87?, 0xc0012a883c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012a86a8 sp=0xc0012a8688 pc=0x4777ae +runtime.selectgo(0xc0012a8890, 0xc0012a8838, 0x7f429a42b200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012a87f8 sp=0xc0012a86a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00128cb40, {0xc0010e5d50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0012a88c0 sp=0xc0012a87f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00128cb40, {0xc0010e5d50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0012a8958 sp=0xc0012a88c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0010e56e0, {0xc0010e5d50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0012a89a0 sp=0xc0012a8958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0010e56e0}, {0xc0010e5d50, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0012a89e8 sp=0xc0012a89a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0010dd440, {0xc0010e5d50, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0012a8a58 sp=0xc0012a89e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0010e5d40, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0012a8b28 sp=0xc0012a8a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0012b4500?, 0xc0010dd440, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0012a8c18 sp=0xc0012a8b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0010e5d40, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc00128d400}, 0xc000c4a360?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0012a8cd0 sp=0xc0012a8c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0010a6780, {0x43dcf60?, 0xc00128d400}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0012a8f08 sp=0xc0012a8cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0010e5e90, {0x43dcf60, 0xc00128d400}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0012a8ff0 sp=0xc0012a8f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000b30aa0, {0x43dcf60, 0xc00128d400}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0012a9028 sp=0xc0012a8ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00128e680) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0012a9068 sp=0xc0012a9028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00128e680?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0012a90a8 sp=0xc0012a9068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00128e690) + :1 +0x46 fp=0xc0012a90e0 sp=0xc0012a90a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c4a4e0, {0x5b51368, 0xc00128e690}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0012a9320 sp=0xc0012a90e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000bfd440, {0x5b51038, 0xc00128e680}) + :1 +0xaf fp=0xc0012a9370 sp=0xc0012a9320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000bfd440}, {0x5b4a6f0?, 0xc000b30aa0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0012a93b0 sp=0xc0012a9370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc0010e5e90}, 0xc0010d8c78?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0012a9400 sp=0xc0012a93b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000bfd440}, {0x5b4ab30, 0xc0010e5e90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0012a9490 sp=0xc0012a9400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc0010a6780}, 0xc0010d8c78, 0xc001283b00) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0012a97d0 sp=0xc0012a9490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000bfd440}, {0x5b492f8, 0xc0010a6780}, 0x3f252a0?, 0xc00128e5b0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0012a9848 sp=0xc0012a97d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0010dd440, 0xc000c4a570, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0012a9db0 sp=0xc0012a9848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0010dd440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0012a9f28 sp=0xc0012a9db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0012a9fe0 sp=0xc0012a9f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012a9fe8 sp=0xc0012a9fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 358 [select]: +runtime.gopark(0xc001291fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001291f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001291de8 sp=0xc001291dc8 pc=0x4777ae +runtime.selectgo(0xc001291fb0, 0xc001291f70, 0xc00128ad20?, 0x0, 0xc0012869c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001291f38 sp=0xc001291de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001291fe0 sp=0xc001291f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001291fe8 sp=0xc001291fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 359 [select]: +runtime.gopark(0xc0012affb0?, 0x2?, 0xb0?, 0xfd?, 0xc0012afee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012afd58 sp=0xc0012afd38 pc=0x4777ae +runtime.selectgo(0xc0012affb0, 0xc0012afee0, 0x448855?, 0x0, 0x4166b20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012afea8 sp=0xc0012afd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0012affe0 sp=0xc0012afea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012affe8 sp=0xc0012affe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 360 [chan receive]: +runtime.gopark(0x7700000000000000?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012aed60 sp=0xc0012aed40 pc=0x4777ae +runtime.chanrecv(0xc001286c60, 0xc0012aee60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0012aedd8 sp=0xc0012aed60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0012aee00 sp=0xc0012aedd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0012aefe0 sp=0xc0012aee00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012aefe8 sp=0xc0012aefe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 361 [select]: +runtime.gopark(0xc0012a4890?, 0x2?, 0x98?, 0x47?, 0xc0012a483c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012a46a8 sp=0xc0012a4688 pc=0x4777ae +runtime.selectgo(0xc0012a4890, 0xc0012a4838, 0x7f429a424300?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0012a47f8 sp=0xc0012a46a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00128d900, {0xc0012b67c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0012a48c0 sp=0xc0012a47f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00128d900, {0xc0012b67c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0012a4958 sp=0xc0012a48c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0012b6720, {0xc0012b67c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0012a49a0 sp=0xc0012a4958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0012b6720}, {0xc0012b67c0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0012a49e8 sp=0xc0012a49a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0010dd9e0, {0xc0012b67c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0012a4a58 sp=0xc0012a49e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0012b67b0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0012a4b28 sp=0xc0012a4a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0012b5a00?, 0xc0010dd9e0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0012a4c18 sp=0xc0012a4b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0012b67b0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc00128dc20}, 0xc000c70240?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0012a4cd0 sp=0xc0012a4c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0010a6870, {0x43dcf60?, 0xc00128dc20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0012a4f08 sp=0xc0012a4cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0012b6900, {0x43dcf60, 0xc00128dc20}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0012a4ff0 sp=0xc0012a4f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000b30ce0, {0x43dcf60, 0xc00128dc20}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0012a5028 sp=0xc0012a4ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00128eaa0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0012a5068 sp=0xc0012a5028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00128eaa0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0012a50a8 sp=0xc0012a5068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00128eab0) + :1 +0x46 fp=0xc0012a50e0 sp=0xc0012a50a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc000c703c0, {0x5b51368, 0xc00128eab0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0012a5320 sp=0xc0012a50e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc000c6e5e0, {0x5b51038, 0xc00128eaa0}) + :1 +0xaf fp=0xc0012a5370 sp=0xc0012a5320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc000c6e5e0}, {0x5b4a6f0?, 0xc000b30ce0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0012a53b0 sp=0xc0012a5370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc0012b6900}, 0xc0010d90e0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0012a5400 sp=0xc0012a53b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b4ab30, 0xc0012b6900}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0012a5490 sp=0xc0012a5400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0010a6870}, 0xc0010d90e0, 0xc0012b4e00) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0012a57d0 sp=0xc0012a5490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc000c6e5e0}, {0x5b492f8, 0xc0010a6870}, 0x3f252a0?, 0xc00128e9d0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0012a5848 sp=0xc0012a57d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc0010dd9e0, 0xc000c70450, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0012a5db0 sp=0xc0012a5848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc000c40d20, {0x5b54e00, 0xc000e8c680}, 0xc0010dd9e0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0012a5f28 sp=0xc0012a5db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0012a5fe0 sp=0xc0012a5f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012a5fe8 sp=0xc0012a5fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 187 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 288 [IO wait]: +runtime.gopark(0xc001140f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001140f40 sp=0xc001140f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001140f78 sp=0xc001140f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42416776f0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001140f98 sp=0xc001140f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d179a0, 0xc000d1d680?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001140fe0 sp=0xc001140f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000d17980, {0xc000d1d680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0011410d0 sp=0xc001140fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc000d17980, {0xc000d1d680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001141130 sp=0xc0011410d0 pc=0x89342b +net.(*conn).Read(0xc000991360, {0xc000d1d680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0011411b8 sp=0xc001141130 pc=0x8b176d +net.(*UnixConn).Read(0xc000991360, {0xc000d1d680, 0x240, 0x240}) + :1 +0x54 fp=0xc001141200 sp=0xc0011411b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc001422108, {0xc000d1d680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001141260 sp=0xc001141200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00125e2a8, {0x5b0f6c0, 0xc001422108}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0011412c8 sp=0xc001141260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00125e000, {0x5b0f840?, 0xc000991360}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001141338 sp=0xc0011412c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00125e000, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001141968 sp=0xc001141338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00125e000, {0xc0013be000, 0x8000, 0x0?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001141a50 sp=0xc001141968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0009d3cc0, {0xc0013be000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001141a98 sp=0xc001141a50 pc=0x10969ed +bufio.(*Reader).Read(0xc0013996e0, {0xc000c49700, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001141b58 sp=0xc001141a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0013996e0}, {0xc000c49700, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001141ba0 sp=0xc001141b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000c49700, 0x9, 0x9}, {0x5b0f0a0, 0xc0013996e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001141c40 sp=0xc001141ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000c496c0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001141d78 sp=0xc001141c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001174340, 0x4b05a9?, 0xc000b81ef8?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001141ec8 sp=0xc001141d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc0004ef680, {0x5b54e00?, 0xc001174340}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001141f80 sp=0xc001141ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001141fe0 sp=0xc001141f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001141fe8 sp=0xc001141fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 304 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 402 [select]: +runtime.gopark(0xc0011d9fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0011d9f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011d9de8 sp=0xc0011d9dc8 pc=0x4777ae +runtime.selectgo(0xc0011d9fb0, 0xc0011d9f70, 0x4b05a9?, 0x0, 0x8d0f55?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011d9f38 sp=0xc0011d9de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0011d9fe0 sp=0xc0011d9f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011d9fe8 sp=0xc0011d9fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 353 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 366 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012b1d60 sp=0xc0012b1d40 pc=0x4777ae +runtime.chanrecv(0xc001287020, 0xc0012b1e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0012b1dd8 sp=0xc0012b1d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0012b1e00 sp=0xc0012b1dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0012b1fe0 sp=0xc0012b1e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012b1fe8 sp=0xc0012b1fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 134 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 409 [select]: +runtime.gopark(0xc00142be48?, 0x2?, 0x0?, 0x0?, 0xc00142bdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029c7c30 sp=0xc0029c7c10 pc=0x4777ae +runtime.selectgo(0xc0029c7e48, 0xc00142bdc8, 0xc00142bdd8?, 0x0, 0xc000dc1500?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029c7d80 sp=0xc0029c7c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001434be0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0029c7e78 sp=0xc0029c7d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000611110) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0029c7f30 sp=0xc0029c7e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0029c7fe0 sp=0xc0029c7f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029c7fe8 sp=0xc0029c7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 379 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 370 [select]: +runtime.gopark(0xc00115ef70?, 0x4?, 0x0?, 0x0?, 0xc00115edd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00115ec00 sp=0xc00115ebe0 pc=0x4777ae +runtime.selectgo(0xc00115ef70, 0xc00115edd0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00115ed50 sp=0xc00115ec00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001174680) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc00115efc0 sp=0xc00115ed50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc00115efe0 sp=0xc00115efc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00115efe8 sp=0xc00115efe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 303 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 371 [IO wait]: +runtime.gopark(0xc001268f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001268f40 sp=0xc001268f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001268f78 sp=0xc001268f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42416778e0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001268f98 sp=0xc001268f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d17820, 0xc000d1d200?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001268fe0 sp=0xc001268f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000d17800, {0xc000d1d200, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0012690d0 sp=0xc001268fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc000d17800, {0xc000d1d200, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001269130 sp=0xc0012690d0 pc=0x89342b +net.(*conn).Read(0xc000991358, {0xc000d1d200, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0012691b8 sp=0xc001269130 pc=0x8b176d +net.(*UnixConn).Read(0xc000991358, {0xc000d1d200, 0x240, 0x240}) + :1 +0x54 fp=0xc001269200 sp=0xc0012691b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0014224c8, {0xc000d1d200, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001269260 sp=0xc001269200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000f89ea8, {0x5b0f6c0, 0xc0014224c8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0012692c8 sp=0xc001269260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000f89c00, {0x5b0f840?, 0xc000991358}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001269338 sp=0xc0012692c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000f89c00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001269968 sp=0xc001269338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000f89c00, {0xc0013d6000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001269a50 sp=0xc001269968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0009d3d40, {0xc0013d6000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001269a98 sp=0xc001269a50 pc=0x10969ed +bufio.(*Reader).Read(0xc001399980, {0xc000c497e0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001269b58 sp=0xc001269a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001399980}, {0xc000c497e0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001269ba0 sp=0xc001269b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000c497e0, 0x9, 0x9}, {0x5b0f0a0, 0xc001399980}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001269c40 sp=0xc001269ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000c497a0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001269d78 sp=0xc001269c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001174680, 0x8d0f0a?, 0x87c25f?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001269ec8 sp=0xc001269d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc0004ef860, {0x5b54e00?, 0xc001174680}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001269f80 sp=0xc001269ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001269fe0 sp=0xc001269f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001269fe8 sp=0xc001269fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 303 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1352 [select]: +runtime.gopark(0xc0022d9f70?, 0x2?, 0xb4?, 0x1b?, 0xc0022d9ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0022d9d20 sp=0xc0022d9d00 pc=0x4777ae +runtime.selectgo(0xc0022d9f70, 0xc0022d9eb8, 0xc001487f80?, 0x0, 0x100000001?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0022d9e70 sp=0xc0022d9d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002180180, {0x5b426e0, 0xc001abb450}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0022d9fa0 sp=0xc0022d9e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0022d9fe0 sp=0xc0022d9fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0022d9fe8 sp=0xc0022d9fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1320 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 580 [select]: +runtime.gopark(0xc0020b8b70?, 0x3?, 0x49?, 0x0?, 0xc001fe007a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001290810 sp=0xc0012907f0 pc=0x4777ae +runtime.selectgo(0xc0020b8b70, 0xc001fe0074, 0x0?, 0x0, 0xc0020843f0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001290960 sp=0xc001290810 pc=0x48916b +reflect.rselect({0xc001290b48, 0x3, 0x447c9e?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:589 +0x345 fp=0xc001290a10 sp=0xc001290960 pc=0x4a8f05 +reflect.Select({0xc001eff7a0, 0x3, 0x2?}) + /home/ec2-user/sdk/go1.21.0/src/reflect/value.go:3104 +0xb8a fp=0xc001290bd8 sp=0xc001290a10 pc=0x57c44a +github.com/rancher/opni/pkg/util.WaitAll({0x5b426e0, 0xc001152370}, 0xc002084180, {0xc001290db8, 0x2, 0x2}) + /home/ec2-user/opni/pkg/util/servers.go:51 +0x3ea fp=0xc001290d58 sp=0xc001290bd8 pc=0x176ae8a +github.com/rancher/opni/pkg/management.(*Server).ListenAndServe(0xc000d8a000, {0x5b426e0, 0xc000b69f40}) + /home/ec2-user/opni/pkg/management/server.go:196 +0x208 fp=0xc001290dd8 sp=0xc001290d58 pc=0x231df08 +github.com/rancher/opni/pkg/test.(*Environment).startGateway.func2(0x4b0571) + /home/ec2-user/opni/pkg/test/environment.go:1699 +0x18c fp=0xc001290f88 sp=0xc001290dd8 pc=0x24c574c +github.com/rancher/opni/pkg/plugins/hooks.onLoadingCompletedHook.Invoke(0xc0007d30e0, 0xc001290fd0?) + /home/ec2-user/opni/pkg/plugins/hooks/complete.go:10 +0x34 fp=0xc001290fa8 sp=0xc001290f88 pc=0x2182b94 +github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete.func1() + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x51 fp=0xc001290fe0 sp=0xc001290fa8 pc=0x2189711 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001290fe8 sp=0xc001290fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete in goroutine 65 + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x138 + +goroutine 373 [select]: +runtime.gopark(0xc00115dfb0?, 0x2?, 0xb4?, 0x1b?, 0xc00115df74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00115dde8 sp=0xc00115ddc8 pc=0x4777ae +runtime.selectgo(0xc00115dfb0, 0xc00115df70, 0x4ad421?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00115df38 sp=0xc00115dde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc00115dfe0 sp=0xc00115df38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00115dfe8 sp=0xc00115dfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 154 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 374 [select]: +runtime.gopark(0xc001115fb0?, 0x2?, 0xb0?, 0x5d?, 0xc001115ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001115d58 sp=0xc001115d38 pc=0x4777ae +runtime.selectgo(0xc001115fb0, 0xc001115ee0, 0x448855?, 0x0, 0x4166b20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001115ea8 sp=0xc001115d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001115fe0 sp=0xc001115ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001115fe8 sp=0xc001115fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 375 [chan receive]: +runtime.gopark(0x7700000000000000?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00115fd60 sp=0xc00115fd40 pc=0x4777ae +runtime.chanrecv(0xc001142f60, 0xc00115fe60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00115fdd8 sp=0xc00115fd60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00115fe00 sp=0xc00115fdd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc00115ffe0 sp=0xc00115fe00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00115ffe8 sp=0xc00115ffe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 154 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 387 [select]: +runtime.gopark(0xc00126bf70?, 0x2?, 0xb4?, 0x1b?, 0xc00126bebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00126bd20 sp=0xc00126bd00 pc=0x4777ae +runtime.selectgo(0xc00126bf70, 0xc00126beb8, 0x4b3745?, 0x0, 0xc000bdcb08?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00126be70 sp=0xc00126bd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b31240, {0x5b426e0, 0xc0012c2820}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc00126bfa0 sp=0xc00126be70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc00126bfe0 sp=0xc00126bfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00126bfe8 sp=0xc00126bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 307 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 388 [select]: +runtime.gopark(0xc00113df70?, 0x2?, 0xb4?, 0x1b?, 0xc00113debc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00113dd20 sp=0xc00113dd00 pc=0x4777ae +runtime.selectgo(0xc00113df70, 0xc00113deb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00113de70 sp=0xc00113dd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b312a0, {0x5b426e0, 0xc0012c2870}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc00113dfa0 sp=0xc00113de70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc00113dfe0 sp=0xc00113dfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00113dfe8 sp=0xc00113dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 307 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 385 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x26?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d1bd0 sp=0xc0018d1bb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0018d1c08 sp=0xc0018d1bd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162ab58, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0018d1c28 sp=0xc0018d1c08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000a3df20, 0xc0019cc000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0018d1c70 sp=0xc0018d1c28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000a3df00, {0xc0019cc000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0018d1d60 sp=0xc0018d1c70 pc=0x50e5a5 +net.(*netFD).Read(0xc000a3df00, {0xc0019cc000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0018d1dc0 sp=0xc0018d1d60 pc=0x89342b +net.(*conn).Read(0xc000b78758, {0xc0019cc000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0018d1e48 sp=0xc0018d1dc0 pc=0x8b176d +net.(*TCPConn).Read(0xc000b78758, {0xc0019cc000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0018d1e90 sp=0xc0018d1e48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc000fe7c00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc0018d1f10 sp=0xc0018d1e90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc000a46000) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc0018d1fc0 sp=0xc0018d1f10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc0018d1fe0 sp=0xc0018d1fc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d1fe8 sp=0xc0018d1fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 177 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 429 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015f5978 sp=0xc0015f5958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0015f59b0 sp=0xc0015f5978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677310, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0015f59d0 sp=0xc0015f59b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00144f020, 0xc0015f5a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0015f5a18 sp=0xc0015f59d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00144f000) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0015f5b08 sp=0xc0015f5a18 pc=0x513f65 +net.(*netFD).accept(0xc00144f000) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0015f5c00 sp=0xc0015f5b08 pc=0x8967be +net.(*UnixListener).accept(0xc0015deab0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc0015f5c40 sp=0xc0015f5c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc0015deab0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc0015f5cb0 sp=0xc0015f5c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc00127db80) + :1 +0x44 fp=0xc0015f5ce8 sp=0xc0015f5cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc00141a3c0, {0x5b2c6a0, 0xc00127db80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0015f5f18 sp=0xc0015f5ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc0015f5f68 sp=0xc0015f5f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc00127de00?, 0xc00158afe0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc0015f5fa8 sp=0xc0015f5f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc0015f5fe0 sp=0xc0015f5fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015f5fe8 sp=0xc0015f5fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 424 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 380 [IO wait]: +runtime.gopark(0xc001498fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001498f98 sp=0xc001498f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001498fd0 sp=0xc001498f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677500, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001498ff0 sp=0xc001498fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000dbbca0, 0xc00139f800?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001499038 sp=0xc001498ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000dbbc80, {0xc00139f800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001499128 sp=0xc001499038 pc=0x50e5a5 +net.(*netFD).Read(0xc000dbbc80, {0xc00139f800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001499188 sp=0xc001499128 pc=0x89342b +net.(*conn).Read(0xc000640318, {0xc00139f800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001499210 sp=0xc001499188 pc=0x8b176d +net.(*UnixConn).Read(0xc000640318, {0xc00139f800, 0x600, 0x600}) + :1 +0x54 fp=0xc001499258 sp=0xc001499210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc001423260, {0xc00139f800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0014992b8 sp=0xc001499258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc1428, {0x5b0f6c0, 0xc001423260}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001499320 sp=0xc0014992b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc1180, {0x5b0f840?, 0xc000640318}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001499390 sp=0xc001499320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc1180, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0014999c0 sp=0xc001499390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc1180, {0xc0014b0000, 0x8000, 0x4?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001499aa8 sp=0xc0014999c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0014922c0, {0xc0014b0000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001499af0 sp=0xc001499aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc0014ac540, {0xc000c498c0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001499bb0 sp=0xc001499af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0014ac540}, {0xc000c498c0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001499bf8 sp=0xc001499bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000c498c0, 0x9, 0x9}, {0x5b0f0a0, 0xc0014ac540}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001499c98 sp=0xc001499bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000c49880) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001499dd0 sp=0xc001499c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001128b40, 0xc0004efe00?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc001499fb0 sp=0xc001499dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc001499fe0 sp=0xc001499fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001499fe8 sp=0xc001499fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 376 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 581 [select]: +runtime.gopark(0xc00164c090?, 0x3?, 0x0?, 0x0?, 0xc001bf808a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001243808 sp=0xc0012437e8 pc=0x4777ae +runtime.selectgo(0xc00164c090, 0xc001bf8084, 0x0?, 0x0, 0xc00163e100?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001243958 sp=0xc001243808 pc=0x48916b +reflect.rselect({0xc001243b40, 0x3, 0x447c9e?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:589 +0x345 fp=0xc001243a08 sp=0xc001243958 pc=0x4a8f05 +reflect.Select({0xc0011d0540, 0x3, 0x2?}) + /home/ec2-user/sdk/go1.21.0/src/reflect/value.go:3104 +0xb8a fp=0xc001243bd0 sp=0xc001243a08 pc=0x57c44a +github.com/rancher/opni/pkg/util.WaitAll({0x5b426e0, 0xc0020885a0}, 0xc00163e090, {0xc001243db8, 0x2, 0x2}) + /home/ec2-user/opni/pkg/util/servers.go:51 +0x3ea fp=0xc001243d50 sp=0xc001243bd0 pc=0x176ae8a +github.com/rancher/opni/pkg/gateway.(*Gateway).ListenAndServe(0xc000912b40, {0x5b426e0, 0xc000b69f40}) + /home/ec2-user/opni/pkg/gateway/gateway.go:347 +0x2ae fp=0xc001243dd8 sp=0xc001243d50 pc=0x22b25ae +github.com/rancher/opni/pkg/test.(*Environment).startGateway.func3(0x4b0571) + /home/ec2-user/opni/pkg/test/environment.go:1709 +0x18c fp=0xc001243f88 sp=0xc001243dd8 pc=0x24c538c +github.com/rancher/opni/pkg/plugins/hooks.onLoadingCompletedHook.Invoke(0xc0007d3100, 0x7f429a42eb00?) + /home/ec2-user/opni/pkg/plugins/hooks/complete.go:10 +0x34 fp=0xc001243fa8 sp=0xc001243f88 pc=0x2182b94 +github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete.func1() + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x51 fp=0xc001243fe0 sp=0xc001243fa8 pc=0x2189711 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001243fe8 sp=0xc001243fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete in goroutine 65 + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x138 + +goroutine 495 [select]: +runtime.gopark(0xc0011ffed0?, 0x2?, 0xb4?, 0x1b?, 0xc0011ffe4c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029c1c90 sp=0xc0029c1c70 pc=0x4777ae +runtime.selectgo(0xc0029c1ed0, 0xc0011ffe48, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029c1de0 sp=0xc0029c1c90 pc=0x48916b +github.com/rancher/opni/pkg/management.(*Server).listenAndServeGrpc(0xc000d8a000, {0x5b426e0, 0xc001152370}) + /home/ec2-user/opni/pkg/management/server.go:215 +0x3cd fp=0xc0029c1f20 sp=0xc0029c1de0 pc=0x231e52d +github.com/rancher/opni/pkg/management.(*Server).ListenAndServe.func1() + /home/ec2-user/opni/pkg/management/server.go:181 +0x4b fp=0xc0029c1fa0 sp=0xc0029c1f20 pc=0x231e0ab +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0029c1fe0 sp=0xc0029c1fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029c1fe8 sp=0xc0029c1fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 580 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 381 [select]: +runtime.gopark(0xc001433e70?, 0x2?, 0x0?, 0x0?, 0xc001433df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029c3c58 sp=0xc0029c3c38 pc=0x4777ae +runtime.selectgo(0xc0029c3e70, 0xc001433df0, 0xc001433e00?, 0x0, 0xc000dc1180?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029c3da8 sp=0xc0029c3c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139d180, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0029c3ea0 sp=0xc0029c3da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0005e4460) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0029c3f58 sp=0xc0029c3ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0029c3fe0 sp=0xc0029c3f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029c3fe8 sp=0xc0029c3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 376 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 403 [select]: +runtime.gopark(0xc00119dfb0?, 0x2?, 0x26?, 0xaf?, 0xc00119dee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00119dd58 sp=0xc00119dd38 pc=0x4777ae +runtime.selectgo(0xc00119dfb0, 0xc00119dee0, 0xe?, 0x0, 0x4b0571?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00119dea8 sp=0xc00119dd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc00119dfe0 sp=0xc00119dea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00119dfe8 sp=0xc00119dfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 353 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 404 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xe0?, 0x4d?, 0x22c5a46?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001114d60 sp=0xc001114d40 pc=0x4777ae +runtime.chanrecv(0xc001436360, 0xc001114e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001114dd8 sp=0xc001114d60 pc=0x441325 +runtime.chanrecv1(0xc00158a1c0?, 0xc00073a6b1?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001114e00 sp=0xc001114dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001114fe0 sp=0xc001114e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001114fe8 sp=0xc001114fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 353 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 410 [select]: +runtime.gopark(0xc001159f70?, 0x4?, 0x9?, 0x0?, 0xc001159dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001159c00 sp=0xc001159be0 pc=0x4777ae +runtime.selectgo(0xc001159f70, 0xc001159dd0, 0xc001159da8?, 0x0, 0xc000f8e380?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001159d50 sp=0xc001159c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00146c9c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001159fc0 sp=0xc001159d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001159fe0 sp=0xc001159fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001159fe8 sp=0xc001159fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 379 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 411 [IO wait]: +runtime.gopark(0xc00126cf88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00126cf40 sp=0xc00126cf20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00126cf78 sp=0xc00126cf40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677408, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00126cf98 sp=0xc00126cf78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000dbbf20, 0xc0014a0000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00126cfe0 sp=0xc00126cf98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000dbbf00, {0xc0014a0000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00126d0d0 sp=0xc00126cfe0 pc=0x50e5a5 +net.(*netFD).Read(0xc000dbbf00, {0xc0014a0000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00126d130 sp=0xc00126d0d0 pc=0x89342b +net.(*conn).Read(0xc000640320, {0xc0014a0000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00126d1b8 sp=0xc00126d130 pc=0x8b176d +net.(*UnixConn).Read(0xc000640320, {0xc0014a0000, 0x240, 0x240}) + :1 +0x54 fp=0xc00126d200 sp=0xc00126d1b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc001423308, {0xc0014a0000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc00126d260 sp=0xc00126d200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc17a8, {0x5b0f6c0, 0xc001423308}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc00126d2c8 sp=0xc00126d260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc1500, {0x5b0f840?, 0xc000640320}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc00126d338 sp=0xc00126d2c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc1500, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc00126d968 sp=0xc00126d338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc1500, {0xc0015be000, 0x8000, 0x4b0751?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc00126da50 sp=0xc00126d968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc00127d5a0, {0xc0015be000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc00126da98 sp=0xc00126da50 pc=0x10969ed +bufio.(*Reader).Read(0xc00127acc0, {0xc000f8e4a0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00126db58 sp=0xc00126da98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00127acc0}, {0xc000f8e4a0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00126dba0 sp=0xc00126db58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000f8e4a0, 0x9, 0x9}, {0x5b0f0a0, 0xc00127acc0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00126dc40 sp=0xc00126dba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000f8e460) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00126dd78 sp=0xc00126dc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00146c9c0, 0x4208e0?, 0x7f429a420400?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc00126dec8 sp=0xc00126dd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc0004efe00, {0x5b54e00?, 0xc00146c9c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc00126df80 sp=0xc00126dec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc00126dfe0 sp=0xc00126df80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00126dfe8 sp=0xc00126dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 379 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 420 [select]: +runtime.gopark(0xc00115af28?, 0x2?, 0xb4?, 0x1b?, 0xc00115aebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00115ad30 sp=0xc00115ad10 pc=0x4777ae +runtime.selectgo(0xc00115af28, 0xc00115aeb8, 0x22c21f5?, 0x0, 0xc00115af20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00115ae80 sp=0xc00115ad30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc0015de0c0, 0xc0014234d0, 0xc00158aba0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc00115af58 sp=0xc00115ae80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI(0xc0015de0c0, {0x5b6f770?, 0xc000d8a000}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:185 +0x10a fp=0xc00115af98 sp=0xc00115af58 pc=0x21a7e0a +github.com/rancher/opni/pkg/management.NewServer.func1.2() + /home/ec2-user/opni/pkg/management/server.go:156 +0x62 fp=0xc00115afe0 sp=0xc00115af98 pc=0x231c0c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00115afe8 sp=0xc00115afe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 417 + /home/ec2-user/opni/pkg/management/server.go:156 +0x137 + +goroutine 421 [select]: +runtime.gopark(0xc00115bf28?, 0x2?, 0xb4?, 0x1b?, 0xc00115bebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00115bd30 sp=0xc00115bd10 pc=0x4777ae +runtime.selectgo(0xc00115bf28, 0xc00115beb8, 0x1f?, 0x0, 0xc00115bf20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00115be80 sp=0xc00115bd30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc0015de0c0, 0xc0014235a8, 0xc00158ac60) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc00115bf58 sp=0xc00115be80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer(0xc0015de0c0, {0x5b21b70?, 0xc000ef5240}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:198 +0x10a fp=0xc00115bf98 sp=0xc00115bf58 pc=0x21a810a +github.com/rancher/opni/pkg/management.NewServer.func1.3() + /home/ec2-user/opni/pkg/management/server.go:158 +0x62 fp=0xc00115bfe0 sp=0xc00115bf98 pc=0x231c022 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00115bfe8 sp=0xc00115bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 417 + /home/ec2-user/opni/pkg/management/server.go:158 +0x245 + +goroutine 649 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc0010e3930?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000c5ddd8 sp=0xc000c5ddb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc001641e50, 0x3a) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc000c5de28 sp=0xc000c5ddd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc001641e40) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc000c5de58 sp=0xc000c5de28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc0020ace00, 0xc0010e3930) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc000c5dfb0 sp=0xc000c5de58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc000c5dfe0 sp=0xc000c5dfb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000c5dfe8 sp=0xc000c5dfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 609 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 425 [select]: +runtime.gopark(0xc000f73448?, 0x2?, 0xd0?, 0x32?, 0xc000f733fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f73278 sp=0xc000f73258 pc=0x4777ae +runtime.selectgo(0xc000f73448, 0xc000f733f8, 0xc000813988?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f733c8 sp=0xc000f73278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00146b560) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc000f73478 sp=0xc000f733c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001471d40, {0x42219c0?, 0xc0015de270}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc000f736d8 sp=0xc000f73478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00146b3d4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc000f73738 sp=0xc000f736d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00146b320, 0xc000f738f0, 0xc000f738c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc000f737e0 sp=0xc000f73738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00146b320, {0x42219c0, 0xc0015de270}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc000f73920 sp=0xc000f737e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc0015de330}, {0x4661da0, 0x1f}, {0x4278de0, 0xc0015de240}, {0x42219c0, 0xc0015de270}, 0x0?, {0xc000996a60, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc000f739c0 sp=0xc000f73920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc000762460}, {0x4661da0, 0x1f}, {0x4278de0, 0xc0015de240}, {0x42219c0, 0xc0015de270}, 0xc000f7a000, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc000f73d30 sp=0xc000f739c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc000f7a000, {0x5b426e0, 0xc000762460}, {0x4661da0, 0x1f}, {0x4278de0, 0xc0015de240}, {0x42219c0, 0xc0015de270}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc000f73e58 sp=0xc000f73d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseManagementAPI(0xc00158aa70, {0x5b426e0, 0xc000762460}, 0x4b0a55?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:51 +0xee fp=0xc000f73ee8 sp=0xc000f73e58 pc=0x21ab66e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI.func2(0x1) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:190 +0xf2 fp=0xc000f73f58 sp=0xc000f73ee8 pc=0x21a7f32 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc000f73fe0 sp=0xc000f73f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f73fe8 sp=0xc000f73fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 420 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 426 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x60?, 0xe0?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001559cd0 sp=0xc001559cb0 pc=0x4777ae +runtime.chanrecv(0xc00127bda0, 0xc001559de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001559d48 sp=0xc001559cd0 pc=0x441325 +runtime.chanrecv1(0xc001559df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001559d70 sp=0xc001559d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001559f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001559e00 sp=0xc001559d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc000ee4410, 0x2?, 0xc0015de510) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001559fa0 sp=0xc001559e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001559fe0 sp=0xc001559fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001559fe8 sp=0xc001559fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 421 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 427 [select]: +runtime.gopark(0xc001165448?, 0x2?, 0xd0?, 0x52?, 0xc0011653fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001165278 sp=0xc001165258 pc=0x4777ae +runtime.selectgo(0xc001165448, 0xc0011653f8, 0xc000813988?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011653c8 sp=0xc001165278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0015f8000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001165478 sp=0xc0011653c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc0015f6000, {0x42219c0?, 0xc0015deb40}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0011656d8 sp=0xc001165478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00146bdf4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001165738 sp=0xc0011656d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00146bd40, 0xc0011658f0, 0xc0011658c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc0011657e0 sp=0xc001165738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00146bd40, {0x42219c0, 0xc0015deb40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001165920 sp=0xc0011657e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc0015dec00}, {0x4673e80, 0x23}, {0x4278de0, 0xc0015deb10}, {0x42219c0, 0xc0015deb40}, 0x0?, {0xc000996a60, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc0011659c0 sp=0xc001165920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc000762460}, {0x4673e80, 0x23}, {0x4278de0, 0xc0015deb10}, {0x42219c0, 0xc0015deb40}, 0xc000f7a000, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc001165d30 sp=0xc0011659c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc000f7a000, {0x5b426e0, 0xc000762460}, {0x4673e80, 0x23}, {0x4278de0, 0xc0015deb10}, {0x42219c0, 0xc0015deb40}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc001165e58 sp=0xc001165d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseNodeManagerClient(0xc00158aa70, {0x5b426e0, 0xc000762460}, 0x4b0a55?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:60 +0xee fp=0xc001165ee8 sp=0xc001165e58 pc=0x21ab7ee +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer.func2(0x2) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:203 +0xf2 fp=0xc001165f58 sp=0xc001165ee8 pc=0x21a8232 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc001165fe0 sp=0xc001165f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001165fe8 sp=0xc001165fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 421 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 430 [select]: +runtime.gopark(0xc000d38f38?, 0x2?, 0x69?, 0x97?, 0xc000d38f14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d38d90 sp=0xc000d38d70 pc=0x4777ae +runtime.selectgo(0xc000d38f38, 0xc000d38f10, 0xc000d38f18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d38ee0 sp=0xc000d38d90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc000d38f68 sp=0xc000d38ee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc0014238d8?, 0xc00158b000?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc000d38fa8 sp=0xc000d38f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc000d38fe0 sp=0xc000d38fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d38fe8 sp=0xc000d38fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 424 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 431 [chan receive]: +runtime.gopark(0x0?, 0x2?, 0x1?, 0x26?, 0xff000000?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00148aef8 sp=0xc00148aed8 pc=0x4777ae +runtime.chanrecv(0xc0010886c0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00148af70 sp=0xc00148aef8 pc=0x441325 +runtime.chanrecv1(0xc000762460?, 0x2106c90?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00148af98 sp=0xc00148af70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc00148afe0 sp=0xc00148af98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00148afe8 sp=0xc00148afe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 426 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 432 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015ef978 sp=0xc0015ef958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0015ef9b0 sp=0xc0015ef978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677218, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0015ef9d0 sp=0xc0015ef9b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00144f1a0, 0xc0015efa01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0015efa18 sp=0xc0015ef9d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00144f180) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0015efb08 sp=0xc0015efa18 pc=0x513f65 +net.(*netFD).accept(0xc00144f180) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0015efc00 sp=0xc0015efb08 pc=0x8967be +net.(*UnixListener).accept(0xc0015deae0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc0015efc40 sp=0xc0015efc00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc0015deae0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc0015efcb0 sp=0xc0015efc40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc00127dbe0) + :1 +0x44 fp=0xc0015efce8 sp=0xc0015efcb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc00141a5a0, {0x5b2c6a0, 0xc00127dbe0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0015eff18 sp=0xc0015efce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc0015eff68 sp=0xc0015eff18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc00127dfe0?, 0xc00158b130?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc0015effa8 sp=0xc0015eff68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc0015effe0 sp=0xc0015effa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015effe8 sp=0xc0015effe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 426 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 433 [select]: +runtime.gopark(0xc00119ef38?, 0x2?, 0x0?, 0x0?, 0xc00119ef14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00119ed90 sp=0xc00119ed70 pc=0x4777ae +runtime.selectgo(0xc00119ef38, 0xc00119ef10, 0xc00119ef18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00119eee0 sp=0xc00119ed90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc00119ef68 sp=0xc00119eee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc001423998?, 0xc00158b150?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc00119efa8 sp=0xc00119ef68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc00119efe0 sp=0xc00119efa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00119efe8 sp=0xc00119efe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 426 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 434 [chan receive]: +runtime.gopark(0xc0007d32a4?, 0x4be6c0?, 0xa0?, 0x32?, 0x4be720?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012d0cf8 sp=0xc0012d0cd8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0012d0d70 sp=0xc0012d0cf8 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0xc001bbbe30?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0012d0d98 sp=0xc0012d0d70 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/gateway.(*Plugin).UseManagementAPI(0xc000cad180, {0x5b6f168, 0xc001661580}) + /home/ec2-user/opni/plugins/metrics/pkg/gateway/system.go:69 +0x67c fp=0xc0012d0fa0 sp=0xc0012d0d98 pc=0x3db3f9c +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseManagementAPI(0xc000e75840, {0x4b05a9?, 0xc00126f0f0?}, 0xc001560180) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:102 +0x30f fp=0xc0012d1098 sp=0xc0012d0fa0 pc=0x21a640f +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler.func1({0x5b426a8, 0xc001560240}, {0x4278de0?, 0xc001560180}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:151 +0x83 fp=0xc0012d10f0 sp=0xc0012d1098 pc=0x21ac063 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc001560240}, {0x4278de0, 0xc001560180}, 0xc001331520, 0xc001321068) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc0012d1148 sp=0xc0012d10f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc001560240}, {0x4278de0, 0xc001560180}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc0012d11e8 sp=0xc0012d1148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc001560150}, {0x4278de0, 0xc001560180}, 0xc001331520, 0xc001367f40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc0012d16b8 sp=0xc0012d11e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc001560150}, {0x4278de0, 0xc001560180}, 0xc00126f778?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc0012d1740 sp=0xc0012d16b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler({0x4278ea0?, 0xc000e75840}, {0x5b426a8, 0xc001560150}, 0xc000611420, 0xc000b74c80) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:153 +0x1c4 fp=0xc0012d17d0 sp=0xc0012d1740 pc=0x21abec4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0015f8120, 0xc000c4af00, 0x771eca0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc0012d1db0 sp=0xc0012d17d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0015f8120, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc0012d1f28 sp=0xc0012d1db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0012d1fe0 sp=0xc0012d1f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012d1fe8 sp=0xc0012d1fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1136 [select]: +runtime.gopark(0xc002f89478?, 0x2?, 0x80?, 0x93?, 0xc002f89414?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002f89280 sp=0xc002f89260 pc=0x4777ae +runtime.selectgo(0xc002f89478, 0xc002f89410, 0xc0020868a0?, 0x0, 0xc002333470?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002f893d0 sp=0xc002f89280 pc=0x48916b +github.com/rancher/opni/pkg/management.(*Server).WatchClusterHealthStatus(0xc0010c0e70, 0x42219c0?, {0x5b4cae0, 0xc001bf00d0}) + /home/ec2-user/opni/pkg/management/health.go:38 +0x352 fp=0xc002f894a8 sp=0xc002f893d0 pc=0x23189f2 +github.com/rancher/opni/pkg/apis/management/v1._Management_WatchClusterHealthStatus_Handler({0x4537040?, 0xc0010c0e70}, {0x5b4ab30, 0xc002086960}) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:829 +0xfa fp=0xc002f89508 sp=0xc002f894a8 pc=0x1781ada +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4537040, 0xc0010c0e70}, {0x5b492f8, 0xc001da4000}, 0xc0028fd2f0, 0x47ec6b8) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002f89848 sp=0xc002f89508 pc=0x12b0e02 +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001ebfc20, {0x5b54e00, 0xc0021d44e0}, 0xc0015f98c0, 0xc001be2420, 0x7716c80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002f89db0 sp=0xc002f89848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001ebfc20, {0x5b54e00, 0xc0021d44e0}, 0xc0015f98c0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002f89f28 sp=0xc002f89db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002f89fe0 sp=0xc002f89f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002f89fe8 sp=0xc002f89fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1007 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 438 [select]: +runtime.gopark(0xc001185f70?, 0x2?, 0xb4?, 0x1b?, 0xc001185ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001185d20 sp=0xc001185d00 pc=0x4777ae +runtime.selectgo(0xc001185f70, 0xc001185eb8, 0xc001175040?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001185e70 sp=0xc001185d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0015fe360, {0x5b426e0, 0xc0015fa730}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001185fa0 sp=0xc001185e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001185fe0 sp=0xc001185fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001185fe8 sp=0xc001185fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 435 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 437 [chan receive]: +runtime.gopark(0x4be6c0?, 0xc000f1df50?, 0x20?, 0xe7?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001726f08 sp=0xc001726ee8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001726f80 sp=0xc001726f08 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0x5b11a20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001726fa8 sp=0xc001726f80 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/gateway.(*Plugin).UseNodeManagerClient(0xc000cad180, {0x5b11a20, 0xc0014d6a50}) + /home/ec2-user/opni/plugins/metrics/pkg/gateway/system.go:74 +0x93 fp=0xc001726fd0 sp=0xc001726fa8 pc=0x3db4593 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseNodeManagerClient(0xc000e75840, {0x4b05a9?, 0xc0017190f0?}, 0xc0015dfe00) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:113 +0x18d fp=0xc001727098 sp=0xc001726fd0 pc=0x21a68cd +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler.func1({0x5b426a8, 0xc0015dfec0}, {0x4278de0?, 0xc0015dfe00}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:169 +0x83 fp=0xc0017270f0 sp=0xc001727098 pc=0x21ac443 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc0015dfec0}, {0x4278de0, 0xc0015dfe00}, 0xc0015fe460, 0xc001423de8) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001727148 sp=0xc0017270f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc0015dfec0}, {0x4278de0, 0xc0015dfe00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc0017271e8 sp=0xc001727148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc0015dfdd0}, {0x4278de0, 0xc0015dfe00}, 0xc0015fe460, 0xc0015d9880) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc0017276b8 sp=0xc0017271e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc0015dfdd0}, {0x4278de0, 0xc0015dfe00}, 0xc001719778?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001727740 sp=0xc0017276b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler({0x4278ea0?, 0xc000e75840}, {0x5b426a8, 0xc0015dfdd0}, 0xc000611500, 0xc000b74c80) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:171 +0x1c4 fp=0xc0017277d0 sp=0xc001727740 pc=0x21ac2a4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0015f8480, 0xc000c4af00, 0x771ecb8, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001727db0 sp=0xc0017277d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0015f8480, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001727f28 sp=0xc001727db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001727fe0 sp=0xc001727f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001727fe8 sp=0xc001727fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 439 [select]: +runtime.gopark(0xc0015f1f70?, 0x2?, 0xb4?, 0x1b?, 0xc0015f1ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015f1d20 sp=0xc0015f1d00 pc=0x4777ae +runtime.selectgo(0xc0015f1f70, 0xc0015f1eb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0015f1e70 sp=0xc0015f1d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0015fe3c0, {0x5b426e0, 0xc0015fa780}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0015f1fa0 sp=0xc0015f1e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0015f1fe0 sp=0xc0015f1fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015f1fe8 sp=0xc0015f1fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 435 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 625 [select]: +runtime.gopark(0xc002ed9e40?, 0x3?, 0xb4?, 0x1b?, 0xc002ed9c2a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ed9a48 sp=0xc002ed9a28 pc=0x4777ae +runtime.selectgo(0xc002ed9e40, 0xc002ed9c24, 0xc002bfaf0b?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ed9b98 sp=0xc002ed9a48 pc=0x48916b +github.com/rancher/opni/pkg/health.(*Monitor).Run(0xc000ee4050, {0x5b426e0, 0xc001fee820}, {0x5b21aa8, 0xc001623400}) + /home/ec2-user/opni/pkg/health/monitor.go:69 +0x1f3 fp=0xc002ed9f80 sp=0xc002ed9b98 pc=0x206e173 +github.com/rancher/opni/pkg/gateway.NewGateway.func23() + /home/ec2-user/opni/pkg/gateway/gateway.go:277 +0x6c fp=0xc002ed9fe0 sp=0xc002ed9f80 pc=0x22aed8c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ed9fe8 sp=0xc002ed9fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.NewGateway in goroutine 609 + /home/ec2-user/opni/pkg/gateway/gateway.go:277 +0x3eed + +goroutine 401 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x26?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a50bd0 sp=0xc000a50bb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000a50c08 sp=0xc000a50bd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a968, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000a50c28 sp=0xc000a50c08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001b16ba0, 0xc001b5e000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000a50c70 sp=0xc000a50c28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001b16b80, {0xc001b5e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000a50d60 sp=0xc000a50c70 pc=0x50e5a5 +net.(*netFD).Read(0xc001b16b80, {0xc001b5e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc000a50dc0 sp=0xc000a50d60 pc=0x89342b +net.(*conn).Read(0xc000bde888, {0xc001b5e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc000a50e48 sp=0xc000a50dc0 pc=0x8b176d +net.(*TCPConn).Read(0xc000bde888, {0xc001b5e000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc000a50e90 sp=0xc000a50e48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc0012b5500) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc000a50f10 sp=0xc000a50e90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc001657c00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc000a50fc0 sp=0xc000a50f10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc000a50fe0 sp=0xc000a50fc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a50fe8 sp=0xc000a50fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 434 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 485 [select]: +runtime.gopark(0xc0014a9e48?, 0x2?, 0x0?, 0x0?, 0xc0014a9dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029cdc30 sp=0xc0029cdc10 pc=0x4777ae +runtime.selectgo(0xc0029cde48, 0xc0014a9dc8, 0xc0014a9dd8?, 0x0, 0xc000dc0a80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029cdd80 sp=0xc0029cdc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139d860, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0029cde78 sp=0xc0029cdd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000611a40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0029cdf30 sp=0xc0029cde78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0029cdfe0 sp=0xc0029cdf30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029cdfe8 sp=0xc0029cdfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 453 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 675 [select]: +runtime.gopark(0xc0017c9e48?, 0x2?, 0x0?, 0x0?, 0xc0017c9dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001127c30 sp=0xc001127c10 pc=0x4777ae +runtime.selectgo(0xc001127e48, 0xc0017c9dc8, 0xc0017c9dd8?, 0x0, 0xc001d6f220?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001127d80 sp=0xc001127c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00099bea0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001127e78 sp=0xc001127d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001d8ce00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001127f30 sp=0xc001127e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc001127fe0 sp=0xc001127f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001127fe8 sp=0xc001127fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 674 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 496 [select]: +runtime.gopark(0xc00151ded0?, 0x2?, 0xb4?, 0x1b?, 0xc00151de1c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029cfc68 sp=0xc0029cfc48 pc=0x4777ae +runtime.selectgo(0xc0029cfed0, 0xc00151de18, 0x1?, 0x0, 0xc001622480?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029cfdb8 sp=0xc0029cfc68 pc=0x48916b +github.com/rancher/opni/pkg/management.(*Server).listenAndServeHttp(0xc000d8a000, {0x5b426e0?, 0xc001152370}) + /home/ec2-user/opni/pkg/management/server.go:248 +0x625 fp=0xc0029cff20 sp=0xc0029cfdb8 pc=0x231ed05 +github.com/rancher/opni/pkg/management.(*Server).ListenAndServe.func2() + /home/ec2-user/opni/pkg/management/server.go:189 +0x4b fp=0xc0029cffa0 sp=0xc0029cff20 pc=0x231dfab +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0029cffe0 sp=0xc0029cffa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029cffe8 sp=0xc0029cffe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 580 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 390 [select]: +runtime.gopark(0xc001715f70?, 0x2?, 0xb4?, 0x1b?, 0xc001715ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001715d20 sp=0xc001715d00 pc=0x4777ae +runtime.selectgo(0xc001715f70, 0xc001715eb8, 0x2322150?, 0x0, 0x23222f5?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001715e70 sp=0xc001715d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001634980, {0x5b426e0, 0xc0016620a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001715fa0 sp=0xc001715e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001715fe0 sp=0xc001715fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001715fe8 sp=0xc001715fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 437 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 391 [select]: +runtime.gopark(0xc0010eff70?, 0x2?, 0xb4?, 0x1b?, 0xc0010efebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010efd20 sp=0xc0010efd00 pc=0x4777ae +runtime.selectgo(0xc0010eff70, 0xc0010efeb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0010efe70 sp=0xc0010efd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0016349e0, {0x5b426e0, 0xc0016620f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0010effa0 sp=0xc0010efe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0010effe0 sp=0xc0010effa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010effe8 sp=0xc0010effe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 437 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 319 [select]: +runtime.gopark(0xc001719f70?, 0x2?, 0xb4?, 0x1b?, 0xc001719ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001719d20 sp=0xc001719d00 pc=0x4777ae +runtime.selectgo(0xc001719f70, 0xc001719eb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001719e70 sp=0xc001719d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001331720, {0x5b426e0, 0xc00137d400}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001719fa0 sp=0xc001719e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001719fe0 sp=0xc001719fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001719fe8 sp=0xc001719fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 434 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 320 [select]: +runtime.gopark(0xc00126ff70?, 0x2?, 0xb4?, 0x1b?, 0xc00126febc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00126fd20 sp=0xc00126fd00 pc=0x4777ae +runtime.selectgo(0xc00126ff70, 0xc00126feb8, 0x472ddb?, 0x0, 0x4ad421?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00126fe70 sp=0xc00126fd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001331780, {0x5b426e0, 0xc00137d450}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc00126ffa0 sp=0xc00126fe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc00126ffe0 sp=0xc00126ffa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00126ffe8 sp=0xc00126ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 434 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1120 [select]: +runtime.gopark(0xc0029bbf50?, 0x2?, 0xb4?, 0x1b?, 0xc0029bbeac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029bbce8 sp=0xc0029bbcc8 pc=0x4777ae +runtime.selectgo(0xc0029bbf50, 0xc0029bbea8, 0xc001305b00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029bbe38 sp=0xc0029bbce8 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).handleDriverNotifications(0xc00070a000) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:227 +0x125 fp=0xc0029bbfc0 sp=0xc0029bbe38 pc=0x3972405 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI.func2() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:89 +0x34 fp=0xc0029bbfe0 sp=0xc0029bbfc0 pc=0x396f0f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029bbfe8 sp=0xc0029bbfe0 pc=0x4ad421 +created by github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI in goroutine 886 + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:89 +0x54b + +goroutine 546 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0x28?, 0x7c?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015e8e18 sp=0xc0015e8df8 pc=0x4777ae +runtime.chanrecv(0xc001089da0, 0xc0015e8ed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0015e8e90 sp=0xc0015e8e18 pc=0x441325 +runtime.chanrecv2(0xc001657c28?, 0x11148f1?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0015e8eb8 sp=0xc0015e8e90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc001657c00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc0015e8fc0 sp=0xc0015e8eb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc0015e8fe0 sp=0xc0015e8fc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015e8fe8 sp=0xc0015e8fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 434 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 454 [IO wait]: +runtime.gopark(0xc00171afe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00171af98 sp=0xc00171af78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00171afd0 sp=0xc00171af98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677120, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00171aff0 sp=0xc00171afd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014d33a0, 0xc001941800?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00171b038 sp=0xc00171aff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0014d3380, {0xc001941800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00171b128 sp=0xc00171b038 pc=0x50e5a5 +net.(*netFD).Read(0xc0014d3380, {0xc001941800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00171b188 sp=0xc00171b128 pc=0x89342b +net.(*conn).Read(0xc000640000, {0xc001941800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00171b210 sp=0xc00171b188 pc=0x8b176d +net.(*UnixConn).Read(0xc000640000, {0xc001941800, 0x3800, 0x3800}) + :1 +0x54 fp=0xc00171b258 sp=0xc00171b210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0004a7ce0, {0xc001941800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc00171b2b8 sp=0xc00171b258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc0628, {0x5b0f6c0, 0xc0004a7ce0}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc00171b320 sp=0xc00171b2b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc0380, {0x5b0f840?, 0xc000640000}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc00171b390 sp=0xc00171b320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc0380, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc00171b9c0 sp=0xc00171b390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc0380, {0xc001010000, 0x8000, 0xc00106e550?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc00171baa8 sp=0xc00171b9c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001492980, {0xc001010000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc00171baf0 sp=0xc00171baa8 pc=0x10969ed +bufio.(*Reader).Read(0xc000f1a7e0, {0xc000886040, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00171bbb0 sp=0xc00171baf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000f1a7e0}, {0xc000886040, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00171bbf8 sp=0xc00171bbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000886040, 0x9, 0x9}, {0x5b0f0a0, 0xc000f1a7e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00171bc98 sp=0xc00171bbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000886000) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00171bdd0 sp=0xc00171bc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001128900, 0xc00141a3c0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc00171bfb0 sp=0xc00171bdd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc00171bfe0 sp=0xc00171bfb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00171bfe8 sp=0xc00171bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 321 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 547 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc0007d3500?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bc1f08 sp=0xc001bc1ee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc0012b54d0, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc001bc1f58 sp=0xc001bc1f08 pc=0x4a9339 +sync.(*Cond).Wait(0xc0012b54c0) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc001bc1f88 sp=0xc001bc1f58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc0007d3500) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc001bc1fc0 sp=0xc001bc1f88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc001bc1fe0 sp=0xc001bc1fc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc1fe8 sp=0xc001bc1fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 434 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 478 [IO wait]: +runtime.gopark(0xc001716fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001716f98 sp=0xc001716f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001716fd0 sp=0xc001716f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241677028, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001716ff0 sp=0xc001716fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00166c5a0, 0xc000f14600?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001717038 sp=0xc001716ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00166c580, {0xc000f14600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001717128 sp=0xc001717038 pc=0x50e5a5 +net.(*netFD).Read(0xc00166c580, {0xc000f14600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001717188 sp=0xc001717128 pc=0x89342b +net.(*conn).Read(0xc0012c6000, {0xc000f14600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001717210 sp=0xc001717188 pc=0x8b176d +net.(*UnixConn).Read(0xc0012c6000, {0xc000f14600, 0x600, 0x600}) + :1 +0x54 fp=0xc001717258 sp=0xc001717210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc001320bb8, {0xc000f14600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0017172b8 sp=0xc001717258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0010b82a8, {0x5b0f6c0, 0xc001320bb8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001717320 sp=0xc0017172b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc0010b8000, {0x5b0f840?, 0xc0012c6000}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001717390 sp=0xc001717320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc0010b8000, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0017179c0 sp=0xc001717390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc0010b8000, {0xc00175e000, 0x8000, 0x4?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001717aa8 sp=0xc0017179c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001635040, {0xc00175e000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001717af0 sp=0xc001717aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc000c4ede0, {0xc000e4ec80, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001717bb0 sp=0xc001717af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000c4ede0}, {0xc000e4ec80, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001717bf8 sp=0xc001717bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000e4ec80, 0x9, 0x9}, {0x5b0f0a0, 0xc000c4ede0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001717c98 sp=0xc001717bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000e4ec40) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001717dd0 sp=0xc001717c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00165a6c0, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc001717fb0 sp=0xc001717dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc001717fe0 sp=0xc001717fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001717fe8 sp=0xc001717fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 392 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 610 [select]: +runtime.gopark(0xc000c5ff70?, 0x2?, 0xb4?, 0x1b?, 0xc000c5febc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000c5fd20 sp=0xc000c5fd00 pc=0x4777ae +runtime.selectgo(0xc000c5ff70, 0xc000c5feb8, 0xc0015fd520?, 0x0, 0x440940?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000c5fe70 sp=0xc000c5fd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001dec3a0, {0x5b426e0, 0xc001152640}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc000c5ffa0 sp=0xc000c5fe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc000c5ffe0 sp=0xc000c5ffa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000c5ffe8 sp=0xc000c5ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 496 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 455 [select]: +runtime.gopark(0xc001633e70?, 0x2?, 0x0?, 0x0?, 0xc001633df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029c9c58 sp=0xc0029c9c38 pc=0x4777ae +runtime.selectgo(0xc0029c9e70, 0xc001633df0, 0xc001633e00?, 0x0, 0xc000dc0380?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029c9da8 sp=0xc0029c9c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001662960, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0029c9ea0 sp=0xc0029c9da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004fe310) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0029c9f58 sp=0xc0029c9ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0029c9fe0 sp=0xc0029c9f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029c9fe8 sp=0xc0029c9fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 321 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1387 [IO wait]: +runtime.gopark(0xc0024d2f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024d2f40 sp=0xc0024d2f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0024d2f78 sp=0xc0024d2f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c48a8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0024d2f98 sp=0xc0024d2f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00257d920, 0xc001156f00?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0024d2fe0 sp=0xc0024d2f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00257d900, {0xc001156f00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0024d30d0 sp=0xc0024d2fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc00257d900, {0xc001156f00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0024d3130 sp=0xc0024d30d0 pc=0x89342b +net.(*conn).Read(0xc001d6f070, {0xc001156f00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0024d31b8 sp=0xc0024d3130 pc=0x8b176d +net.(*UnixConn).Read(0xc001d6f070, {0xc001156f00, 0x500, 0x500}) + :1 +0x54 fp=0xc0024d3200 sp=0xc0024d31b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00304a2e8, {0xc001156f00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0024d3260 sp=0xc0024d3200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00133b428, {0x5b0f6c0, 0xc00304a2e8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0024d32c8 sp=0xc0024d3260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00133b180, {0x5b0f840?, 0xc001d6f070}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0024d3338 sp=0xc0024d32c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00133b180, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0024d3968 sp=0xc0024d3338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00133b180, {0xc00282e000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0024d3a50 sp=0xc0024d3968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d86e60, {0xc00282e000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0024d3a98 sp=0xc0024d3a50 pc=0x10969ed +bufio.(*Reader).Read(0xc002175980, {0xc0014f90e0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0024d3b58 sp=0xc0024d3a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002175980}, {0xc0014f90e0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0024d3ba0 sp=0xc0024d3b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0014f90e0, 0x9, 0x9}, {0x5b0f0a0, 0xc002175980}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0024d3c40 sp=0xc0024d3ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0014f90a0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0024d3d78 sp=0xc0024d3c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0024e7860, 0xc001c6a0f0?, 0x5b2ad00?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc0024d3ec8 sp=0xc0024d3d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc0018534a0, {0x5b54e00?, 0xc0024e7860}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc0024d3f80 sp=0xc0024d3ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc0024d3fe0 sp=0xc0024d3f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024d3fe8 sp=0xc0024d3fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1394 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 396 [select]: +runtime.gopark(0xc0015afe48?, 0x2?, 0x0?, 0x0?, 0xc0015afdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029cbc30 sp=0xc0029cbc10 pc=0x4777ae +runtime.selectgo(0xc0029cbe48, 0xc0015afdc8, 0xc0015afdd8?, 0x0, 0xc000dc0700?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029cbd80 sp=0xc0029cbc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00137dc70, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0029cbe78 sp=0xc0029cbd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0006e6070) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0029cbf30 sp=0xc0029cbe78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0029cbfe0 sp=0xc0029cbf30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029cbfe8 sp=0xc0029cbfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 452 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 397 [select]: +runtime.gopark(0xc0015ebf70?, 0x4?, 0x38?, 0xdc?, 0xc0015ebdd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015ebc00 sp=0xc0015ebbe0 pc=0x4777ae +runtime.selectgo(0xc0015ebf70, 0xc0015ebdd0, 0x40a16a0?, 0x0, 0xc000e9ca20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0015ebd50 sp=0xc0015ebc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000c7c000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc0015ebfc0 sp=0xc0015ebd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc0015ebfe0 sp=0xc0015ebfc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015ebfe8 sp=0xc0015ebfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 452 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 398 [IO wait]: +runtime.gopark(0xc001534f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001534f40 sp=0xc001534f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001534f78 sp=0xc001534f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162ac50, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001534f98 sp=0xc001534f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014d36a0, 0xc0014a0900?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001534fe0 sp=0xc001534f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0014d3680, {0xc0014a0900, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0015350d0 sp=0xc001534fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc0014d3680, {0xc0014a0900, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001535130 sp=0xc0015350d0 pc=0x89342b +net.(*conn).Read(0xc000640010, {0xc0014a0900, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0015351b8 sp=0xc001535130 pc=0x8b176d +net.(*UnixConn).Read(0xc000640010, {0xc0014a0900, 0x240, 0x240}) + :1 +0x54 fp=0xc001535200 sp=0xc0015351b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc001320bd0, {0xc0014a0900, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001535260 sp=0xc001535200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc09a8, {0x5b0f6c0, 0xc001320bd0}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0015352c8 sp=0xc001535260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc0700, {0x5b0f840?, 0xc000640010}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001535338 sp=0xc0015352c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc0700, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001535968 sp=0xc001535338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc0700, {0xc001456000, 0x8000, 0x0?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001535a50 sp=0xc001535968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001330ac0, {0xc001456000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001535a98 sp=0xc001535a50 pc=0x10969ed +bufio.(*Reader).Read(0xc0012494a0, {0xc000d5c040, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001535b58 sp=0xc001535a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0012494a0}, {0xc000d5c040, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001535ba0 sp=0xc001535b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000d5c040, 0x9, 0x9}, {0x5b0f0a0, 0xc0012494a0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001535c40 sp=0xc001535ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000d5c000) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001535d78 sp=0xc001535c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000c7c000, 0x0?, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001535ec8 sp=0xc001535d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00141a5a0, {0x5b54e00?, 0xc000c7c000}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001535f80 sp=0xc001535ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001535fe0 sp=0xc001535f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001535fe8 sp=0xc001535fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 452 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 456 [select]: +runtime.gopark(0xc001486f20?, 0x2?, 0xb4?, 0x1b?, 0xc001486eb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001486d28 sp=0xc001486d08 pc=0x4777ae +runtime.selectgo(0xc001486f20, 0xc001486eb0, 0x0?, 0x0, 0xc001486f18?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001486e78 sp=0xc001486d28 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc0015de0c0, 0xc000fe1bd0, 0xc000fe1be0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001486f50 sp=0xc001486e78 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore(0xc0015de0c0, {0x5b43370?, 0xc000fe1bc0}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:214 +0x1a8 fp=0xc001486f98 sp=0xc001486f50 pc=0x21a84a8 +github.com/rancher/opni/pkg/gateway.NewGateway.func2.1() + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x62 fp=0xc001486fe0 sp=0xc001486f98 pc=0x22b18c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001486fe8 sp=0xc001486fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.NewGateway.func2 in goroutine 419 + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x3cc + +goroutine 459 [chan receive]: +runtime.gopark(0x78?, 0x7769700?, 0x70?, 0x5f?, 0x11782b6?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00148cef8 sp=0xc00148ced8 pc=0x4777ae +runtime.chanrecv(0xc0010886c0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00148cf70 sp=0xc00148cef8 pc=0x441325 +runtime.chanrecv1(0xc000762460?, 0x5b43370?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00148cf98 sp=0xc00148cf70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc00148cfe0 sp=0xc00148cf98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00148cfe8 sp=0xc00148cfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 457 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 460 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00149b978 sp=0xc00149b958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00149b9b0 sp=0xc00149b978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6e818, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00149b9d0 sp=0xc00149b9b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014d3f20, 0xc00149ba01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00149ba18 sp=0xc00149b9d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0014d3f00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc00149bb08 sp=0xc00149ba18 pc=0x513f65 +net.(*netFD).accept(0xc0014d3f00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc00149bc00 sp=0xc00149bb08 pc=0x8967be +net.(*UnixListener).accept(0xc000ff9080) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc00149bc40 sp=0xc00149bc00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc000ff9080) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc00149bcb0 sp=0xc00149bc40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc001492dc0) + :1 +0x44 fp=0xc00149bce8 sp=0xc00149bcb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc0010a4d20, {0x5b2c6a0, 0xc001492dc0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc00149bf18 sp=0xc00149bce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc00149bf68 sp=0xc00149bf18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc001492f20?, 0xc000fe1f40?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc00149bfa8 sp=0xc00149bf68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc00149bfe0 sp=0xc00149bfa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00149bfe8 sp=0xc00149bfe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 457 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 461 [select]: +runtime.gopark(0xc0011daf38?, 0x2?, 0x0?, 0x0?, 0xc0011daf14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011dad90 sp=0xc0011dad70 pc=0x4777ae +runtime.selectgo(0xc0011daf38, 0xc0011daf10, 0xc0011daf18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011daee0 sp=0xc0011dad90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc0011daf68 sp=0xc0011daee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc00149e990?, 0xc000fe1f70?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc0011dafa8 sp=0xc0011daf68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc0011dafe0 sp=0xc0011dafa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011dafe8 sp=0xc0011dafe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 457 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 462 [chan receive]: +runtime.gopark(0xc0007d3304?, 0x4be6c0?, 0x0?, 0x33?, 0x4be720?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000bacdd0 sp=0xc000bacdb0 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc000bace48 sp=0xc000bacdd0 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0xc000ee6320?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc000bace70 sp=0xc000bace48 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/gateway.(*Plugin).UseKeyValueStore(0xc000cad180, {0x5b43f90?, 0xc0018be840}) + /home/ec2-user/opni/plugins/metrics/pkg/gateway/system.go:91 +0x610 fp=0xc000bacfd0 sp=0xc000bace70 pc=0x3db4bf0 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseKeyValueStore(0xc000e75840, {0x4b05a9?, 0xc0015310f0?}, 0xc001150810) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:123 +0x18d fp=0xc000bad098 sp=0xc000bacfd0 pc=0x21a6c2d +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler.func1({0x5b426a8, 0xc001150930}, {0x4278de0?, 0xc001150810}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:187 +0x83 fp=0xc000bad0f0 sp=0xc000bad098 pc=0x21ac823 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc001150930}, {0x4278de0, 0xc001150810}, 0xc001493360, 0xc00149ed38) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc000bad148 sp=0xc000bad0f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc001150930}, {0x4278de0, 0xc001150810}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc000bad1e8 sp=0xc000bad148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc0011507e0}, {0x4278de0, 0xc001150810}, 0xc001493360, 0xc0014d9900) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc000bad6b8 sp=0xc000bad1e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc0011507e0}, {0x4278de0, 0xc001150810}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc000bad740 sp=0xc000bad6b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler({0x4278ea0?, 0xc000e75840}, {0x5b426a8, 0xc0011507e0}, 0xc0007d1ea0, 0xc000b74c80) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:189 +0x1c4 fp=0xc000bad7d0 sp=0xc000bad740 pc=0x21ac684 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0010dcd80, 0xc000c4af00, 0x771ecd0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc000baddb0 sp=0xc000bad7d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc000c401e0, {0x5b54e00, 0xc000c7cb60}, 0xc0010dcd80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc000badf28 sp=0xc000baddb0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000badfe0 sp=0xc000badf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000badfe8 sp=0xc000badfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 184 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 600 [select]: +runtime.gopark(0xc000d06ed8?, 0x2?, 0xb4?, 0x1b?, 0xc000d06e84?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d06cf8 sp=0xc000d06cd8 pc=0x4777ae +runtime.selectgo(0xc000d06ed8, 0xc000d06e80, 0xc000d06ea8?, 0x0, 0xc000d06e90?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d06e48 sp=0xc000d06cf8 pc=0x48916b +github.com/rancher/opni/pkg/util.ServeHandler({0x5b426e0, 0xc001fee370}, {0x5b13c00?, 0xc000102ea0}, {0x5b2a6d0?, 0xc001f0e540}) + /home/ec2-user/opni/pkg/util/servers.go:21 +0x26a fp=0xc000d06f08 sp=0xc000d06e48 pc=0x176a94a +github.com/rancher/opni/pkg/gateway.(*GatewayHTTPServer).ListenAndServe.func2() + /home/ec2-user/opni/pkg/gateway/http.go:182 +0x1b9 fp=0xc000d06fa0 sp=0xc000d06f08 pc=0x22b8699 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc000d06fe0 sp=0xc000d06fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d06fe8 sp=0xc000d06fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 558 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 464 [select]: +runtime.gopark(0xc001533f70?, 0x2?, 0xb4?, 0x1b?, 0xc001533ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001533d20 sp=0xc001533d00 pc=0x4777ae +runtime.selectgo(0xc001533f70, 0xc001533eb8, 0x4aead9?, 0x0, 0x215774d?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001533e70 sp=0xc001533d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001493560, {0x5b426e0, 0xc0016635e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001533fa0 sp=0xc001533e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001533fe0 sp=0xc001533fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001533fe8 sp=0xc001533fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 462 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 465 [select]: +runtime.gopark(0xc001531f70?, 0x2?, 0xb4?, 0x1b?, 0xc001531ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001531d20 sp=0xc001531d00 pc=0x4777ae +runtime.selectgo(0xc001531f70, 0xc001531eb8, 0x4aead9?, 0x0, 0x5d5276?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001531e70 sp=0xc001531d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0014935c0, {0x5b426e0, 0xc001663630}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001531fa0 sp=0xc001531e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001531fe0 sp=0xc001531fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001531fe8 sp=0xc001531fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 462 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 601 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00148d9e0 sp=0xc00148d9c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00148da18 sp=0xc00148d9e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a490, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00148da38 sp=0xc00148da18 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0019e7fa0, 0x4b0501?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00148da80 sp=0xc00148da38 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0019e7f80) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc00148db70 sp=0xc00148da80 pc=0x513f65 +net.(*netFD).accept(0xc0019e7f80) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc00148dc68 sp=0xc00148db70 pc=0x8967be +net.(*TCPListener).accept(0xc001f0e540) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc00148dcb0 sp=0xc00148dc68 pc=0x8c383e +net.(*TCPListener).Accept(0xc001f0e540) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc00148dd20 sp=0xc00148dcb0 pc=0x8c1ba5 +net/http.(*onceCloseListener).Accept(0xc0017dd200) + :1 +0x44 fp=0xc00148dd58 sp=0xc00148dd20 pc=0xab5544 +net/http.(*Server).Serve(0xc001da40f0, {0x5b2a6d0, 0xc001f0e540}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc00148df50 sp=0xc00148dd58 pc=0xa7859f +github.com/rancher/opni/pkg/util.ServeHandler.func1() + /home/ec2-user/opni/pkg/util/servers.go:19 +0x47 fp=0xc00148dfa0 sp=0xc00148df50 pc=0x176aa67 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc00148dfe0 sp=0xc00148dfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00148dfe8 sp=0xc00148dfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 600 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 497 [select]: +runtime.gopark(0xc000c61f70?, 0x2?, 0xb4?, 0x1b?, 0xc000c61ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000c61d20 sp=0xc000c61d00 pc=0x4777ae +runtime.selectgo(0xc000c61f70, 0xc000c61eb8, 0xc000442070?, 0x0, 0x5b426e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000c61e70 sp=0xc000c61d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001dec2c0, {0x5b426e0, 0xc0011524b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc000c61fa0 sp=0xc000c61e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc000c61fe0 sp=0xc000c61fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000c61fe8 sp=0xc000c61fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 496 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1225 [select]: +runtime.gopark(0xc001e33f10?, 0x3?, 0xb4?, 0x1b?, 0xc001e33e92?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e33ca8 sp=0xc001e33c88 pc=0x4777ae +runtime.selectgo(0xc001e33f10, 0xc001e33e8c, 0x5b126c0?, 0x0, 0x1?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e33df8 sp=0xc001e33ca8 pc=0x48916b +google.golang.org/grpc.(*addrConn).resetTransport(0xc000ebd500) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:1368 +0x3ef fp=0xc001e33f50 sp=0xc001e33df8 pc=0x116334f +google.golang.org/grpc.(*addrConn).connect(0xc000ebd500) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:993 +0x23b fp=0xc001e33fc0 sp=0xc001e33f50 pc=0x116033b +google.golang.org/grpc.(*acBalancerWrapper).Connect.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0x34 fp=0xc001e33fe0 sp=0xc001e33fc0 pc=0x1157434 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e33fe8 sp=0xc001e33fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*acBalancerWrapper).Connect in goroutine 542 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0xa7 + +goroutine 383 [select]: +runtime.gopark(0xc0018d2f80?, 0x3?, 0xb4?, 0x1b?, 0xc0018d2ee2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d2d30 sp=0xc0018d2d10 pc=0x4777ae +runtime.selectgo(0xc0018d2f80, 0xc0018d2edc, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0018d2e80 sp=0xc0018d2d30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc001106180) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:52 +0x178 fp=0xc0018d2fc0 sp=0xc0018d2e80 pc=0x13b0398 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc0018d2fe0 sp=0xc0018d2fc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d2fe8 sp=0xc0018d2fe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 307 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 514 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0x28?, 0x60?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d3e18 sp=0xc0018d3df8 pc=0x4777ae +runtime.chanrecv(0xc00119bb00, 0xc0018d3ed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0018d3e90 sp=0xc0018d3e18 pc=0x441325 +runtime.chanrecv2(0xc000a46028?, 0xc00119ba40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0018d3eb8 sp=0xc0018d3e90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc000a46000) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc0018d3fc0 sp=0xc0018d3eb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc0018d3fe0 sp=0xc0018d3fc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d3fe8 sp=0xc0018d3fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 177 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 515 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc0015fff80?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019fcf08 sp=0xc0019fcee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc000fe7bd0, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc0019fcf58 sp=0xc0019fcf08 pc=0x4a9339 +sync.(*Cond).Wait(0xc000fe7bc0) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc0019fcf88 sp=0xc0019fcf58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc0015fff80) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc0019fcfc0 sp=0xc0019fcf88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc0019fcfe0 sp=0xc0019fcfc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019fcfe8 sp=0xc0019fcfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 177 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 516 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019fdf08 sp=0xc0019fdee8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0019fdf80 sp=0xc0019fdf08 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0xc0019db7d0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0019fdfa8 sp=0xc0019fdf80 pc=0x440f72 +github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore.func3() + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:99 +0x4a fp=0xc0019fdfe0 sp=0xc0019fdfa8 pc=0x24a06aa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019fdfe8 sp=0xc0019fdfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore in goroutine 177 + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:98 +0x4cb + +goroutine 517 [select]: +runtime.gopark(0xc00119bc90?, 0x13b0513?, 0x0?, 0x1?, 0xc0018d4ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d4d30 sp=0xc0018d4d10 pc=0x4777ae +runtime.selectgo(0xc0018d4f60, 0xc0018d4ec8, 0x0?, 0x1, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0018d4e80 sp=0xc0018d4d30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc0011069c0) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:64 +0x318 fp=0xc0018d4fc0 sp=0xc0018d4e80 pc=0x13b0538 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc0018d4fe0 sp=0xc0018d4fc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d4fe8 sp=0xc0018d4fe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 177 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 518 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x26?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d0bd0 sp=0xc0018d0bb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0018d0c08 sp=0xc0018d0bd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162aa60, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0018d0c28 sp=0xc0018d0c08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000a3dfa0, 0xc0019dc000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0018d0c70 sp=0xc0018d0c28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000a3df80, {0xc0019dc000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0018d0d60 sp=0xc0018d0c70 pc=0x50e5a5 +net.(*netFD).Read(0xc000a3df80, {0xc0019dc000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0018d0dc0 sp=0xc0018d0d60 pc=0x89342b +net.(*conn).Read(0xc000b78778, {0xc0019dc000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0018d0e48 sp=0xc0018d0dc0 pc=0x8b176d +net.(*TCPConn).Read(0xc000b78778, {0xc0019dc000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0018d0e90 sp=0xc0018d0e48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc000fe7e00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc0018d0f10 sp=0xc0018d0e90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc000a47c00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc0018d0fc0 sp=0xc0018d0f10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc0018d0fe0 sp=0xc0018d0fc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d0fe8 sp=0xc0018d0fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 307 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 519 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0x28?, 0x7c?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019fbe18 sp=0xc0019fbdf8 pc=0x4777ae +runtime.chanrecv(0xc00119bda0, 0xc0019fbed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0019fbe90 sp=0xc0019fbe18 pc=0x441325 +runtime.chanrecv2(0xc000a47c28?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0019fbeb8 sp=0xc0019fbe90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc000a47c00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc0019fbfc0 sp=0xc0019fbeb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc0019fbfe0 sp=0xc0019fbfc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019fbfe8 sp=0xc0019fbfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 307 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 520 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc000b741c0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019f7f08 sp=0xc0019f7ee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc000fe7dd0, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc0019f7f58 sp=0xc0019f7f08 pc=0x4a9339 +sync.(*Cond).Wait(0xc000fe7dc0) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc0019f7f88 sp=0xc0019f7f58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc000b741c0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc0019f7fc0 sp=0xc0019f7f88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc0019f7fe0 sp=0xc0019f7fc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019f7fe8 sp=0xc0019f7fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 307 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 521 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc00123bc70?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018dddd8 sp=0xc0018dddb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc00130c2d0, 0x15) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc0018dde28 sp=0xc0018dddd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc00130c2c0) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc0018dde58 sp=0xc0018dde28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc000a47c00, 0xc00123bc70) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc0018ddfb0 sp=0xc0018dde58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc0018ddfe0 sp=0xc0018ddfb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018ddfe8 sp=0xc0018ddfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 307 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 522 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc0019fe0d0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018dfdd8 sp=0xc0018dfdb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc00130c890, 0xa) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc0018dfe28 sp=0xc0018dfdd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc00130c880) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc0018dfe58 sp=0xc0018dfe28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc000a46000, 0xc0019fe0d0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc0018dffb0 sp=0xc0018dfe58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc0018dffe0 sp=0xc0018dffb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018dffe8 sp=0xc0018dffe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 177 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 502 [IO wait]: +runtime.gopark(0xc00152efe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00152ef98 sp=0xc00152ef78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00152efd0 sp=0xc00152ef98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6e530, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00152eff0 sp=0xc00152efd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0018bc920, 0xc000f15800?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00152f038 sp=0xc00152eff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0018bc900, {0xc000f15800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00152f128 sp=0xc00152f038 pc=0x50e5a5 +net.(*netFD).Read(0xc0018bc900, {0xc000f15800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00152f188 sp=0xc00152f128 pc=0x89342b +net.(*conn).Read(0xc000640cc0, {0xc000f15800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00152f210 sp=0xc00152f188 pc=0x8b176d +net.(*UnixConn).Read(0xc000640cc0, {0xc000f15800, 0x600, 0x600}) + :1 +0x54 fp=0xc00152f258 sp=0xc00152f210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0001267c8, {0xc000f15800, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc00152f2b8 sp=0xc00152f258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc10a8, {0x5b0f6c0, 0xc0001267c8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc00152f320 sp=0xc00152f2b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc0e00, {0x5b0f840?, 0xc000640cc0}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc00152f390 sp=0xc00152f320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc0e00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc00152f9c0 sp=0xc00152f390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc0e00, {0xc001ad0000, 0x8000, 0xc000830550?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc00152faa8 sp=0xc00152f9c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0005fac00, {0xc001ad0000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc00152faf0 sp=0xc00152faa8 pc=0x10969ed +bufio.(*Reader).Read(0xc00128b0e0, {0xc001ae8040, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00152fbb0 sp=0xc00152faf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00128b0e0}, {0xc001ae8040, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00152fbf8 sp=0xc00152fbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001ae8040, 0x9, 0x9}, {0x5b0f0a0, 0xc00128b0e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00152fc98 sp=0xc00152fbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001ae8000) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00152fdd0 sp=0xc00152fc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000f94480, 0xc00128b200?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc00152ffb0 sp=0xc00152fdd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc00152ffe0 sp=0xc00152ffb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00152ffe8 sp=0xc00152ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 498 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 503 [select]: +runtime.gopark(0xc001af1e70?, 0x2?, 0x0?, 0x0?, 0xc001af1df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029d5c58 sp=0xc0029d5c38 pc=0x4777ae +runtime.selectgo(0xc0029d5e70, 0xc001af1df0, 0xc001af1e00?, 0x0, 0xc000dc0e00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029d5da8 sp=0xc0029d5c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000763e00, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0029d5ea0 sp=0xc0029d5da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001aab260) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0029d5f58 sp=0xc0029d5ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0029d5fe0 sp=0xc0029d5f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029d5fe8 sp=0xc0029d5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 498 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 634 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000186830 sp=0xc000186810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000186868 sp=0xc000186830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a778, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000186888 sp=0xc000186868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001651aa0, 0xc002066000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0001868d0 sp=0xc000186888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001651a80, {0xc002066000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0001869c0 sp=0xc0001868d0 pc=0x50e5a5 +net.(*netFD).Read(0xc001651a80, {0xc002066000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc000186a20 sp=0xc0001869c0 pc=0x89342b +net.(*conn).Read(0xc001d6e0c0, {0xc002066000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc000186aa8 sp=0xc000186a20 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6e0c0, {0xc002066000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc000186af0 sp=0xc000186aa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc001f16ae0, {0xc0011d0660, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc000186bb0 sp=0xc000186af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001f16ae0}, {0xc0011d0660, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000186bf8 sp=0xc000186bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d0660, 0x9, 0x9}, {0x5b0f0a0, 0xc001f16ae0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc000186c98 sp=0xc000186bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d0620) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc000186dd0 sp=0xc000186c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0014546c0, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc000186fb0 sp=0xc000186dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc000186fe0 sp=0xc000186fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000186fe8 sp=0xc000186fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 611 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 526 [select]: +runtime.gopark(0xc001bb7e48?, 0x2?, 0x0?, 0x0?, 0xc001bb7dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029d7c30 sp=0xc0029d7c10 pc=0x4777ae +runtime.selectgo(0xc0029d7e48, 0xc001bb7dc8, 0xc001bb7dd8?, 0x0, 0xc000dc1880?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029d7d80 sp=0xc0029d7c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0015fbc70, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0029d7e78 sp=0xc0029d7d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0008b6fc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0029d7f30 sp=0xc0029d7e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0029d7fe0 sp=0xc0029d7f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029d7fe8 sp=0xc0029d7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 501 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 528 [IO wait]: +runtime.gopark(0xc001ba6f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ba6f40 sp=0xc001ba6f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001ba6f78 sp=0xc001ba6f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4299d6e340, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001ba6f98 sp=0xc001ba6f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0018bcc20, 0xc0014a1680?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001ba6fe0 sp=0xc001ba6f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0018bcc00, {0xc0014a1680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001ba70d0 sp=0xc001ba6fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc0018bcc00, {0xc0014a1680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001ba7130 sp=0xc001ba70d0 pc=0x89342b +net.(*conn).Read(0xc000640d18, {0xc0014a1680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001ba71b8 sp=0xc001ba7130 pc=0x8b176d +net.(*UnixConn).Read(0xc000640d18, {0xc0014a1680, 0x240, 0x240}) + :1 +0x54 fp=0xc001ba7200 sp=0xc001ba71b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc000126798, {0xc0014a1680, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001ba7260 sp=0xc001ba7200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000dc1b28, {0x5b0f6c0, 0xc000126798}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001ba72c8 sp=0xc001ba7260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000dc1880, {0x5b0f840?, 0xc000640d18}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001ba7338 sp=0xc001ba72c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000dc1880, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001ba7968 sp=0xc001ba7338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000dc1880, {0xc001b80000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001ba7a50 sp=0xc001ba7968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc000b74b60, {0xc001b80000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001ba7a98 sp=0xc001ba7a50 pc=0x10969ed +bufio.(*Reader).Read(0xc001107b60, {0xc0011d4740, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001ba7b58 sp=0xc001ba7a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001107b60}, {0xc0011d4740, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001ba7ba0 sp=0xc001ba7b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d4740, 0x9, 0x9}, {0x5b0f0a0, 0xc001107b60}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001ba7c40 sp=0xc001ba7ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d4700) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001ba7d78 sp=0xc001ba7c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0019e4820, 0x0?, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001ba7ec8 sp=0xc001ba7d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc0010a4d20, {0x5b54e00?, 0xc0019e4820}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001ba7f80 sp=0xc001ba7ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001ba7fe0 sp=0xc001ba7f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ba7fe8 sp=0xc001ba7fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 501 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 545 [select]: +runtime.gopark(0xc001ddff70?, 0x2?, 0xb4?, 0x1b?, 0xc001ddfebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ddfd20 sp=0xc001ddfd00 pc=0x4777ae +runtime.selectgo(0xc001ddff70, 0xc001ddfeb8, 0xc00123f860?, 0x0, 0x440100?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ddfe70 sp=0xc001ddfd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000ee7860, {0x5b426e0, 0xc00010ff90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001ddffa0 sp=0xc001ddfe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001ddffe0 sp=0xc001ddffa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ddffe8 sp=0xc001ddffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 434 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 543 [select]: +runtime.gopark(0xc0018dbf70?, 0x2?, 0xb4?, 0x1b?, 0xc0018dbebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018dbd20 sp=0xc0018dbd00 pc=0x4777ae +runtime.selectgo(0xc0018dbf70, 0xc0018dbeb8, 0x0?, 0x0, 0x4aef8c?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0018dbe70 sp=0xc0018dbd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000ee7500, {0x5b426e0, 0xc00010f900}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0018dbfa0 sp=0xc0018dbe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0018dbfe0 sp=0xc0018dbfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018dbfe8 sp=0xc0018dbfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 434 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 562 [select]: +runtime.gopark(0xc001ba5f70?, 0x2?, 0xb4?, 0x1b?, 0xc001ba5ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ba5d20 sp=0xc001ba5d00 pc=0x4777ae +runtime.selectgo(0xc001ba5f70, 0xc001ba5eb8, 0xc0019e4ea0?, 0x0, 0x440940?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ba5e70 sp=0xc001ba5d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000ee78e0, {0x5b426e0, 0xc000b52000}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001ba5fa0 sp=0xc001ba5e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001ba5fe0 sp=0xc001ba5fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ba5fe8 sp=0xc001ba5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 434 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 548 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bc6f08 sp=0xc001bc6ee8 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001bc6f80 sp=0xc001bc6f08 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0x1134db0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001bc6fa8 sp=0xc001bc6f80 pc=0x440f72 +github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore.func3() + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:99 +0x4a fp=0xc001bc6fe0 sp=0xc001bc6fa8 pc=0x24a06aa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc6fe8 sp=0xc001bc6fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore in goroutine 434 + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:98 +0x4cb + +goroutine 549 [select]: +runtime.gopark(0xc001b70070?, 0x13b0513?, 0x0?, 0x1?, 0xc001bc7ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bc7d30 sp=0xc001bc7d10 pc=0x4777ae +runtime.selectgo(0xc001bc7f60, 0xc001bc7ec8, 0x0?, 0x1, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bc7e80 sp=0xc001bc7d30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc0012c8480) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:64 +0x318 fp=0xc001bc7fc0 sp=0xc001bc7e80 pc=0x13b0538 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc001bc7fe0 sp=0xc001bc7fc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc7fe8 sp=0xc001bc7fe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 434 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 550 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc001170dd0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001579dd8 sp=0xc001579db8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc0012b5710, 0x9) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc001579e28 sp=0xc001579dd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc0012b5700) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc001579e58 sp=0xc001579e28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc001657c00, 0xc001170dd0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc001579fb0 sp=0xc001579e58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc001579fe0 sp=0xc001579fb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001579fe8 sp=0xc001579fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 434 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 551 [select]: +runtime.gopark(0xc001bc2f50?, 0x2?, 0xb4?, 0x1b?, 0xc001bc2eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bc2ce8 sp=0xc001bc2cc8 pc=0x4777ae +runtime.selectgo(0xc001bc2f50, 0xc001bc2ea8, 0xc001bc2fa0?, 0x0, 0xc001170dd0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bc2e38 sp=0xc001bc2ce8 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).handleDriverNotifications(0xc0005a8c80) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:227 +0x125 fp=0xc001bc2fc0 sp=0xc001bc2e38 pc=0x3972405 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI.func2() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:89 +0x34 fp=0xc001bc2fe0 sp=0xc001bc2fc0 pc=0x396f0f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc2fe8 sp=0xc001bc2fe0 pc=0x4ad421 +created by github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI in goroutine 177 + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:89 +0x54b + +goroutine 552 [select]: +runtime.gopark(0xc0020dbf40?, 0x3?, 0xb4?, 0x1b?, 0xc0020dbd8a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020dbb90 sp=0xc0020dbb70 pc=0x4777ae +runtime.selectgo(0xc0020dbf40, 0xc0020dbd84, 0x5?, 0x0, 0x40ed7c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020dbce0 sp=0xc0020dbb90 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).runSync(0xc0005a8c80) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/admin.go:530 +0x80f fp=0xc0020dbfc0 sp=0xc0020dbce0 pc=0x396034f +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI.func3() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:90 +0x34 fp=0xc0020dbfe0 sp=0xc0020dbfc0 pc=0x396f094 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020dbfe8 sp=0xc0020dbfe0 pc=0x4ad421 +created by github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI in goroutine 177 + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:90 +0x5cb + +goroutine 1272 [select]: +runtime.gopark(0xc000a2ff10?, 0x3?, 0xb4?, 0x1b?, 0xc000a2fe92?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a2fca8 sp=0xc000a2fc88 pc=0x4777ae +runtime.selectgo(0xc000a2ff10, 0xc000a2fe8c, 0x5b126c0?, 0x0, 0x1?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a2fdf8 sp=0xc000a2fca8 pc=0x48916b +google.golang.org/grpc.(*addrConn).resetTransport(0xc000ebd800) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:1368 +0x3ef fp=0xc000a2ff50 sp=0xc000a2fdf8 pc=0x116334f +google.golang.org/grpc.(*addrConn).connect(0xc000ebd800) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:993 +0x23b fp=0xc000a2ffc0 sp=0xc000a2ff50 pc=0x116033b +google.golang.org/grpc.(*acBalancerWrapper).Connect.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0x34 fp=0xc000a2ffe0 sp=0xc000a2ffc0 pc=0x1157434 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a2ffe8 sp=0xc000a2ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*acBalancerWrapper).Connect in goroutine 545 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0xa7 + +goroutine 564 [chan receive]: +runtime.gopark(0xc000b69f58?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bc5f00 sp=0xc001bc5ee0 pc=0x4777ae +runtime.chanrecv(0xc000982120, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001bc5f78 sp=0xc001bc5f00 pc=0x441325 +runtime.chanrecv1(0xc000b69f40?, 0xc000b69f40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001bc5fa0 sp=0xc001bc5f78 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/cortex.NewClientSet.func1() + /home/ec2-user/opni/plugins/metrics/pkg/cortex/clientset.go:555 +0x53 fp=0xc001bc5fe0 sp=0xc001bc5fa0 pc=0x3d9cfb3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc5fe8 sp=0xc001bc5fe0 pc=0x4ad421 +created by github.com/rancher/opni/plugins/metrics/pkg/cortex.NewClientSet in goroutine 434 + /home/ec2-user/opni/plugins/metrics/pkg/cortex/clientset.go:554 +0x999 + +goroutine 566 [select]: +runtime.gopark(0xc0011e5890?, 0x2?, 0x18?, 0x57?, 0xc0011e5844?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00157d6c0 sp=0xc00157d6a0 pc=0x4777ae +runtime.selectgo(0xc00157d890, 0xc0011e5840, 0xc001de8340?, 0x0, 0x100000001?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00157d810 sp=0xc00157d6c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001dee000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc00157d8c0 sp=0xc00157d810 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001de8270, {0x42d9ca0?, 0xc0017e1200}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc00157db20 sp=0xc00157d8c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001d8bdf4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc00157db80 sp=0xc00157db20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001d8bd40, 0xc00157dd38, 0xc00157dd08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc00157dc28 sp=0xc00157db80 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8bd40, {0x42d9ca0, 0xc0017e1200}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc00157dd68 sp=0xc00157dc28 pc=0x119077e +github.com/rancher/opni/pkg/apis/management/v1.(*managementWatchClusterHealthStatusClient).Recv(0xc001dc4c10) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:252 +0x66 fp=0xc00157dda8 sp=0xc00157dd68 pc=0x177d086 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).watchGlobalClusterHealthStatus(0xc0005a8c80, {0x5b6f168?, 0xc000fe1300}, 0xc001b9d2c0) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/management.go:228 +0x642 fp=0xc00157df68 sp=0xc00157dda8 pc=0x3965de2 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).useWatchers.func2() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:247 +0x1be fp=0xc00157dfc0 sp=0xc00157df68 pc=0x3972fbe +github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents.func1() + /home/ec2-user/opni/pkg/management/watcher.go:27 +0x2f fp=0xc00157dfe0 sp=0xc00157dfc0 pc=0x23220cf +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00157dfe8 sp=0xc00157dfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents in goroutine 177 + /home/ec2-user/opni/pkg/management/watcher.go:26 +0x54 + +goroutine 567 [select]: +runtime.gopark(0xc0020e3f50?, 0x2?, 0xb4?, 0x1b?, 0xc0020e3e0c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020e3c48 sp=0xc0020e3c28 pc=0x4777ae +runtime.selectgo(0xc0020e3f50, 0xc0020e3e08, 0xc001dea380?, 0x0, 0xc0009d3ee0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020e3d98 sp=0xc0020e3c48 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).watchCortexClusterStatus(0xc0005a8c80) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/management.go:70 +0x556 fp=0xc0020e3fa0 sp=0xc0020e3d98 pc=0x3963d76 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).useWatchers.func3() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:248 +0x2f fp=0xc0020e3fc0 sp=0xc0020e3fa0 pc=0x3972dcf +github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents.func1() + /home/ec2-user/opni/pkg/management/watcher.go:27 +0x2f fp=0xc0020e3fe0 sp=0xc0020e3fc0 pc=0x23220cf +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020e3fe8 sp=0xc0020e3fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents in goroutine 177 + /home/ec2-user/opni/pkg/management/watcher.go:26 +0x54 + +goroutine 568 [select]: +runtime.gopark(0xc000097fb0?, 0x2?, 0xb4?, 0x1b?, 0xc000097f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000097de8 sp=0xc000097dc8 pc=0x4777ae +runtime.selectgo(0xc000097fb0, 0xc000097f70, 0x1f75e37?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000097f38 sp=0xc000097de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc000097fe0 sp=0xc000097f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000097fe8 sp=0xc000097fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 565 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 569 [chan receive]: +runtime.gopark(0xc001519300?, 0xc001bd56e0?, 0x0?, 0x0?, 0xc0017dd080?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001519318 sp=0xc0015192f8 pc=0x4777ae +runtime.chanrecv(0xc001f16480, 0xc001519478, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001519390 sp=0xc001519318 pc=0x441325 +runtime.chanrecv2(0xc000bdc140?, 0x5b426a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0015193b8 sp=0xc001519390 pc=0x440f92 +github.com/rancher/opni/pkg/management.(*Server).WatchClusters(0xc000d8a000, 0xc0017dd200, {0x5b4cb30, 0xc001bf1430}) + /home/ec2-user/opni/pkg/management/clusters.go:93 +0x26d fp=0xc0015194a8 sp=0xc0015193b8 pc=0x230a3cd +github.com/rancher/opni/pkg/apis/management/v1._Management_WatchClusters_Handler({0x4537040?, 0xc000d8a000}, {0x5b4ab30, 0xc0017dd1d0}) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:736 +0xfa fp=0xc001519508 sp=0xc0015194a8 pc=0x178093a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4537040, 0xc000d8a000}, {0x5b492f8, 0xc001b9d3b0}, 0xc000127c68, 0x47ec6c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001519848 sp=0xc001519508 pc=0x12b0e02 +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef860, {0x5b54e00, 0xc001174680}, 0xc0019cbe60, 0xc000ff8960, 0x7716c60, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001519db0 sp=0xc001519848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef860, {0x5b54e00, 0xc001174680}, 0xc0019cbe60, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001519f28 sp=0xc001519db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001519fe0 sp=0xc001519f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001519fe8 sp=0xc001519fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 371 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 570 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc001f1a000?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0008b0dd8 sp=0xc0008b0db8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc0017e0890, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc0008b0e28 sp=0xc0008b0dd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc0017e0880) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc0008b0e58 sp=0xc0008b0e28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc000c77100, 0xc001f1a000) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc0008b0fb0 sp=0xc0008b0e58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc0008b0fe0 sp=0xc0008b0fb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008b0fe8 sp=0xc0008b0fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 569 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 571 [chan receive]: +runtime.gopark(0x3fa3c40?, 0xc001bd5860?, 0x0?, 0x0?, 0xc0019faf00?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019faf08 sp=0xc0019faee8 pc=0x4777ae +runtime.chanrecv(0xc001bd5860, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0019faf80 sp=0xc0019faf08 pc=0x441325 +runtime.chanrecv1(0xc001152190?, 0xc000c77100?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0019fafa8 sp=0xc0019faf80 pc=0x440f72 +github.com/nats-io/nats%2ego.(*js).subscribe.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:1817 +0x4a fp=0xc0019fafe0 sp=0xc0019fafa8 pc=0x1f4624a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019fafe8 sp=0xc0019fafe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*js).subscribe in goroutine 569 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:1816 +0x2c85 + +goroutine 572 [select]: +runtime.gopark(0xc000a4ef90?, 0x2?, 0x50?, 0xed?, 0xc000a4ee94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a4ece8 sp=0xc000a4ecc8 pc=0x4777ae +runtime.selectgo(0xc000a4ef90, 0xc000a4ee90, 0x5b11a20?, 0x0, 0xc00081e1c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a4ee38 sp=0xc000a4ece8 pc=0x48916b +github.com/rancher/opni/pkg/storage/jetstream.(*JetStreamStore).WatchClusters.func1() + /home/ec2-user/opni/pkg/storage/jetstream/cluster_store.go:247 +0x18c fp=0xc000a4efe0 sp=0xc000a4ee38 pc=0x249e4ac +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a4efe8 sp=0xc000a4efe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.(*JetStreamStore).WatchClusters in goroutine 569 + /home/ec2-user/opni/pkg/storage/jetstream/cluster_store.go:244 +0xbcd + +goroutine 670 [IO wait]: +runtime.gopark(0xc0024c7bf0?, 0x26?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024c7bd0 sp=0xc0024c7bb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0024c7c08 sp=0xc0024c7bd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241629dc8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0024c7c28 sp=0xc0024c7c08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014c8aa0, 0xc00188c000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0024c7c70 sp=0xc0024c7c28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0014c8a80, {0xc00188c000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0024c7d60 sp=0xc0024c7c70 pc=0x50e5a5 +net.(*netFD).Read(0xc0014c8a80, {0xc00188c000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0024c7dc0 sp=0xc0024c7d60 pc=0x89342b +net.(*conn).Read(0xc001ce24d8, {0xc00188c000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0024c7e48 sp=0xc0024c7dc0 pc=0x8b176d +net.(*TCPConn).Read(0xc001ce24d8, {0xc00188c000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0024c7e90 sp=0xc0024c7e48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc001641d00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc0024c7f10 sp=0xc0024c7e90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc0020ace00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc0024c7fc0 sp=0xc0024c7f10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc0024c7fe0 sp=0xc0024c7fc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024c7fe8 sp=0xc0024c7fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 609 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 599 [select]: +runtime.gopark(0xc0008b1ee0?, 0x2?, 0xb4?, 0x1b?, 0xc0008b1e8c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0008b1d00 sp=0xc0008b1ce0 pc=0x4777ae +runtime.selectgo(0xc0008b1ee0, 0xc0008b1e88, 0x9?, 0x0, 0x5b426e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0008b1e50 sp=0xc0008b1d00 pc=0x48916b +github.com/rancher/opni/pkg/util.ServeHandler({0x5b426e0, 0xc001fee370}, {0x5b13c00?, 0xc000102b60}, {0x5b2a5b0?, 0xc0010d99b0}) + /home/ec2-user/opni/pkg/util/servers.go:21 +0x26a fp=0xc0008b1f10 sp=0xc0008b1e50 pc=0x176a94a +github.com/rancher/opni/pkg/gateway.(*GatewayHTTPServer).ListenAndServe.func1() + /home/ec2-user/opni/pkg/gateway/http.go:178 +0x199 fp=0xc0008b1fa0 sp=0xc0008b1f10 pc=0x22b8879 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0008b1fe0 sp=0xc0008b1fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0008b1fe8 sp=0xc0008b1fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 558 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 593 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0016f3830 sp=0xc0016f3810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0016f3868 sp=0xc0016f3830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a0b0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0016f3888 sp=0xc0016f3868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014c8920, 0xc001f50000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0016f38d0 sp=0xc0016f3888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0014c8900, {0xc001f50000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0016f39c0 sp=0xc0016f38d0 pc=0x50e5a5 +net.(*netFD).Read(0xc0014c8900, {0xc001f50000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0016f3a20 sp=0xc0016f39c0 pc=0x89342b +net.(*conn).Read(0xc000b79250, {0xc001f50000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0016f3aa8 sp=0xc0016f3a20 pc=0x8b176d +net.(*TCPConn).Read(0xc000b79250, {0xc001f50000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0016f3af0 sp=0xc0016f3aa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc001655920, {0xc0011d4900, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0016f3bb0 sp=0xc0016f3af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001655920}, {0xc0011d4900, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0016f3bf8 sp=0xc0016f3bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d4900, 0x9, 0x9}, {0x5b0f0a0, 0xc001655920}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0016f3c98 sp=0xc0016f3bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d48c0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0016f3dd0 sp=0xc0016f3c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001285200, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0016f3fb0 sp=0xc0016f3dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0016f3fe0 sp=0xc0016f3fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0016f3fe8 sp=0xc0016f3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 587 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 613 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015e9988 sp=0xc0015e9968 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0015e99c0 sp=0xc0015e9988 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a680, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0015e99e0 sp=0xc0015e99c0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0019e7f20, 0xc000d8fb01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0015e9a28 sp=0xc0015e99e0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0019e7f00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0015e9b18 sp=0xc0015e9a28 pc=0x513f65 +net.(*netFD).accept(0xc0019e7f00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0015e9c10 sp=0xc0015e9b18 pc=0x8967be +net.(*TCPListener).accept(0xc001f0e4a0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc0015e9c58 sp=0xc0015e9c10 pc=0x8c383e +net.(*TCPListener).Accept(0xc001f0e4a0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc0015e9cc8 sp=0xc0015e9c58 pc=0x8c1ba5 +crypto/tls.(*listener).Accept(0xc0010d99b0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/tls.go:66 +0x43 fp=0xc0015e9d20 sp=0xc0015e9cc8 pc=0x9b2d43 +net/http.(*onceCloseListener).Accept(0xc0020b8ff0) + :1 +0x44 fp=0xc0015e9d58 sp=0xc0015e9d20 pc=0xab5544 +net/http.(*Server).Serve(0xc001b9c000, {0x5b2a5b0, 0xc0010d99b0}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc0015e9f50 sp=0xc0015e9d58 pc=0xa7859f +github.com/rancher/opni/pkg/util.ServeHandler.func1() + /home/ec2-user/opni/pkg/util/servers.go:19 +0x47 fp=0xc0015e9fa0 sp=0xc0015e9f50 pc=0x176aa67 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0015e9fe0 sp=0xc0015e9fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015e9fe8 sp=0xc0015e9fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 599 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 729 [select]: +runtime.gopark(0xc0016efda8?, 0x2?, 0x60?, 0xf8?, 0xc0016efa94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0016ef818 sp=0xc0016ef7f8 pc=0x4777ae +runtime.selectgo(0xc0016efda8, 0xc0016efa90, 0x0?, 0x0, 0x1860a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0016ef968 sp=0xc0016ef818 pc=0x48916b +github.com/hashicorp/go-plugin.Serve(0xc0010fe060) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:474 +0x2091 fp=0xc0016effc0 sp=0xc0016ef968 pc=0x21665b1 +github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins.func4() + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x34 fp=0xc0016effe0 sp=0xc0016effc0 pc=0x24cd234 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0016effe8 sp=0xc0016effe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins in goroutine 609 + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x76b + +goroutine 635 [select]: +runtime.gopark(0xc001f4fe70?, 0x2?, 0x0?, 0x0?, 0xc001f4fdf4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011e5c58 sp=0xc0011e5c38 pc=0x4777ae +runtime.selectgo(0xc0011e5e70, 0xc001f4fdf0, 0xc001f4fe00?, 0x0, 0xc001d6e0c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011e5da8 sp=0xc0011e5c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002088820, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0011e5ea0 sp=0xc0011e5da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0005e5570) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0011e5f58 sp=0xc0011e5ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0011e5fe0 sp=0xc0011e5f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011e5fe8 sp=0xc0011e5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 611 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 615 [select]: +runtime.gopark(0xc001055e48?, 0x2?, 0x0?, 0x0?, 0xc001055dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00121dc30 sp=0xc00121dc10 pc=0x4777ae +runtime.selectgo(0xc00121de48, 0xc001055dc8, 0xc001055dd8?, 0x0, 0xc000990210?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00121dd80 sp=0xc00121dc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001152c80, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00121de78 sp=0xc00121dd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001bf21c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00121df30 sp=0xc00121de78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc00121dfe0 sp=0xc00121df30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00121dfe8 sp=0xc00121dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 614 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 616 [select]: +runtime.gopark(0xc0015eaf70?, 0x4?, 0x28?, 0xd8?, 0xc0015eadd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015eac00 sp=0xc0015eabe0 pc=0x4777ae +runtime.selectgo(0xc0015eaf70, 0xc0015eadd0, 0x40a16a0?, 0x0, 0xc00173ad50?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0015ead50 sp=0xc0015eac00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001aa9a00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc0015eafc0 sp=0xc0015ead50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc0015eafe0 sp=0xc0015eafc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015eafe8 sp=0xc0015eafe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 614 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 617 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d077d8 sp=0xc000d077b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000d07810 sp=0xc000d077d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a398, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000d07830 sp=0xc000d07810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0016228a0, 0xc000f34000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000d07878 sp=0xc000d07830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001622880, {0xc000f34000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000d07968 sp=0xc000d07878 pc=0x50e5a5 +net.(*netFD).Read(0xc001622880, {0xc000f34000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc000d079c8 sp=0xc000d07968 pc=0x89342b +net.(*conn).Read(0xc000990210, {0xc000f34000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc000d07a50 sp=0xc000d079c8 pc=0x8b176d +net.(*TCPConn).Read(0xc000990210, {0xc000f34000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc000d07a98 sp=0xc000d07a50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc001df9020, {0xc001effa80, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc000d07b58 sp=0xc000d07a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001df9020}, {0xc001effa80, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000d07ba0 sp=0xc000d07b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001effa80, 0x9, 0x9}, {0x5b0f0a0, 0xc001df9020}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc000d07c40 sp=0xc000d07ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001effa40) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc000d07d78 sp=0xc000d07c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001aa9a00, 0x4208e0?, 0x7f429a430900?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc000d07ec8 sp=0xc000d07d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc000e0c3c0, {0x5b54e00?, 0xc001aa9a00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc000d07f80 sp=0xc000d07ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc000d07fe0 sp=0xc000d07f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d07fe8 sp=0xc000d07fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 614 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 606 [IO wait]: +runtime.gopark(0x0?, 0xc00172ab40?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020e5708 sp=0xc0020e56e8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0020e5740 sp=0xc0020e5708 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a2a0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0020e5760 sp=0xc0020e5740 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d72120, 0xc0011aa000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0020e57a8 sp=0xc0020e5760 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000d72100, {0xc0011aa000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0020e5898 sp=0xc0020e57a8 pc=0x50e5a5 +net.(*netFD).Read(0xc000d72100, {0xc0011aa000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0020e58f8 sp=0xc0020e5898 pc=0x89342b +net.(*conn).Read(0xc001ce2290, {0xc0011aa000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0020e5980 sp=0xc0020e58f8 pc=0x8b176d +net.(*TCPConn).Read(0xc001ce2290, {0xc0011aa000, 0x1000, 0x1000}) + :1 +0x54 fp=0xc0020e59c8 sp=0xc0020e5980 pc=0x8d72f4 +net/http.(*persistConn).Read(0xc001f186c0, {0xc0011aa000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1954 +0xf9 fp=0xc0020e5a48 sp=0xc0020e59c8 pc=0xa9c1d9 +bufio.(*Reader).fill(0xc001e9b9e0) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:113 +0x29a fp=0xc0020e5ad8 sp=0xc0020e5a48 pc=0x5f863a +bufio.(*Reader).Peek(0xc001e9b9e0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:151 +0xc7 fp=0xc0020e5b60 sp=0xc0020e5ad8 pc=0x5f88e7 +net/http.(*persistConn).readLoop(0xc001f186c0) + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:2118 +0x2db fp=0xc0020e5fc0 sp=0xc0020e5b60 pc=0xa9d41b +net/http.(*Transport).dialConn.func5() + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1776 +0x34 fp=0xc0020e5fe0 sp=0xc0020e5fc0 pc=0xa9aef4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020e5fe8 sp=0xc0020e5fe0 pc=0x4ad421 +created by net/http.(*Transport).dialConn in goroutine 586 + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1776 +0x24da + +goroutine 605 [IO wait]: +runtime.gopark(0xc001ba1650?, 0x4d1aca?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ba15e0 sp=0xc001ba15c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001ba1618 sp=0xc001ba15e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424162a1a8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001ba1638 sp=0xc001ba1618 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc000d73220, 0xc001e01000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001ba1680 sp=0xc001ba1638 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000d73200, {0xc001e01000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001ba1770 sp=0xc001ba1680 pc=0x50e5a5 +net.(*netFD).Read(0xc000d73200, {0xc001e01000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001ba17d0 sp=0xc001ba1770 pc=0x89342b +net.(*conn).Read(0xc001ce2288, {0xc001e01000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001ba1858 sp=0xc001ba17d0 pc=0x8b176d +net.(*TCPConn).Read(0xc001ce2288, {0xc001e01000, 0x1000, 0x1000}) + :1 +0x54 fp=0xc001ba18a0 sp=0xc001ba1858 pc=0x8d72f4 +net/http.(*connReader).Read(0xc0017dd920, {0xc001e01000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:791 +0x2b5 fp=0xc001ba1938 sp=0xc001ba18a0 pc=0xa68115 +bufio.(*Reader).fill(0xc001e9b980) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:113 +0x29a fp=0xc001ba19c8 sp=0xc001ba1938 pc=0x5f863a +bufio.(*Reader).Peek(0xc001e9b980, 0x4) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:151 +0xc7 fp=0xc001ba1a50 sp=0xc001ba19c8 pc=0x5f88e7 +net/http.(*conn).serve(0xc00172a2d0, {0x5b426a8, 0xc0017dd2f0}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2044 +0xe7c fp=0xc001ba1fa0 sp=0xc001ba1a50 pc=0xa7221c +net/http.(*Server).Serve.func3() + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3086 +0x50 fp=0xc001ba1fe0 sp=0xc001ba1fa0 pc=0xa78af0 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ba1fe8 sp=0xc001ba1fe0 pc=0x4ad421 +created by net/http.(*Server).Serve in goroutine 601 + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3086 +0x80d + +goroutine 607 [select]: +runtime.gopark(0xc001555f90?, 0x2?, 0x10?, 0x5d?, 0xc001555e6c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029d1cc8 sp=0xc0029d1ca8 pc=0x4777ae +runtime.selectgo(0xc0029d1f90, 0xc001555e68, 0xc001fdac00?, 0x0, 0xc0017dd9e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029d1e18 sp=0xc0029d1cc8 pc=0x48916b +net/http.(*persistConn).writeLoop(0xc001f186c0) + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:2421 +0x196 fp=0xc0029d1fc0 sp=0xc0029d1e18 pc=0xa9fe36 +net/http.(*Transport).dialConn.func6() + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1777 +0x34 fp=0xc0029d1fe0 sp=0xc0029d1fc0 pc=0xa9ae94 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029d1fe8 sp=0xc0029d1fe0 pc=0x4ad421 +created by net/http.(*Transport).dialConn in goroutine 586 + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1777 +0x256b + +goroutine 642 [select]: +runtime.gopark(0xc00111fe70?, 0x2?, 0x0?, 0x0?, 0xc00111fdf4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001555c58 sp=0xc001555c38 pc=0x4777ae +runtime.selectgo(0xc001555e70, 0xc00111fdf0, 0xc00111fe00?, 0x0, 0xc000b79250?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001555da8 sp=0xc001555c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00137c820, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001555ea0 sp=0xc001555da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0007f9b20) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001555f58 sp=0xc001555ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc001555fe0 sp=0xc001555f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001555fe8 sp=0xc001555fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 587 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 643 [select]: +runtime.gopark(0xc0011c5e48?, 0x2?, 0x0?, 0x0?, 0xc0011c5dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f2fc30 sp=0xc001f2fc10 pc=0x4777ae +runtime.selectgo(0xc001f2fe48, 0xc0011c5dc8, 0xc0005e56c0?, 0x0, 0xc000b79248?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f2fd80 sp=0xc001f2fc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00137c910, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001f2fe78 sp=0xc001f2fd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0005e56c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001f2ff30 sp=0xc001f2fe78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc001f2ffe0 sp=0xc001f2ff30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f2ffe8 sp=0xc001f2ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 591 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 644 [select]: +runtime.gopark(0xc0018d6f70?, 0x4?, 0x10?, 0x0?, 0xc0018d6dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d6c00 sp=0xc0018d6be0 pc=0x4777ae +runtime.selectgo(0xc0018d6f70, 0xc0018d6dd0, 0xc0018d6da8?, 0x0, 0xc0011d48c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0018d6d50 sp=0xc0018d6c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0012a16c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc0018d6fc0 sp=0xc0018d6d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc0018d6fe0 sp=0xc0018d6fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d6fe8 sp=0xc0018d6fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 591 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 645 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000c5b7d8 sp=0xc000c5b7b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000c5b810 sp=0xc000c5b7d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241629fb8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000c5b830 sp=0xc000c5b810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014c89a0, 0xc001f68000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000c5b878 sp=0xc000c5b830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0014c8980, {0xc001f68000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000c5b968 sp=0xc000c5b878 pc=0x50e5a5 +net.(*netFD).Read(0xc0014c8980, {0xc001f68000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc000c5b9c8 sp=0xc000c5b968 pc=0x89342b +net.(*conn).Read(0xc000b79248, {0xc001f68000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc000c5ba50 sp=0xc000c5b9c8 pc=0x8b176d +net.(*TCPConn).Read(0xc000b79248, {0xc001f68000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc000c5ba98 sp=0xc000c5ba50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc001655a40, {0xc0011d49e0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc000c5bb58 sp=0xc000c5ba98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001655a40}, {0xc0011d49e0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000c5bba0 sp=0xc000c5bb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d49e0, 0x9, 0x9}, {0x5b0f0a0, 0xc001655a40}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc000c5bc40 sp=0xc000c5bba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d49a0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc000c5bd78 sp=0xc000c5bc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0012a16c0, 0x2c7d7d22676e6972?, 0x6f69746964646122?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc000c5bec8 sp=0xc000c5bd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc000e0c3c0, {0x5b54e00?, 0xc0012a16c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc000c5bf80 sp=0xc000c5bec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc000c5bfe0 sp=0xc000c5bf80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000c5bfe8 sp=0xc000c5bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 591 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 658 [IO wait]: +runtime.gopark(0x4?, 0xc0012abc30?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012abc48 sp=0xc0012abc28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0012abc80 sp=0xc0012abc48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4241629ec0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0012abca0 sp=0xc0012abc80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001646140, 0xc001800000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0012abce8 sp=0xc0012abca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001646120, {0xc001800000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0012abdd8 sp=0xc0012abce8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001ce2378, {0xc001800000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc0012abe38 sp=0xc0012abdd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc000013200}, {0x5b0eb40, 0xc001ce2378}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc0012abee0 sp=0xc0012abe38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc0012abf50 sp=0xc0012abee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc001f0ecc0) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc0012abfb8 sp=0xc0012abf50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc0012abfe0 sp=0xc0012abfb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012abfe8 sp=0xc0012abfe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 609 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 659 [IO wait]: +runtime.gopark(0x4b05a9?, 0xc0012adc70?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0012adc48 sp=0xc0012adc28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0012adc80 sp=0xc0012adc48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b390, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0012adca0 sp=0xc0012adc80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001646200, 0xc001814000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0012adce8 sp=0xc0012adca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0016461e0, {0xc001814000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0012addd8 sp=0xc0012adce8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001ce2390, {0xc001814000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc0012ade38 sp=0xc0012addd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc000013338}, {0x5b0eb40, 0xc001ce2390}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc0012adee0 sp=0xc0012ade38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc0012adf50 sp=0xc0012adee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc001f0ed00) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc0012adfb8 sp=0xc0012adf50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc0012adfe0 sp=0xc0012adfb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0012adfe8 sp=0xc0012adfe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 609 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 660 [select]: +runtime.gopark(0xc001bd4910?, 0x656a30?, 0x0?, 0x1?, 0xc00205be7c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00205bce0 sp=0xc00205bcc0 pc=0x4777ae +runtime.selectgo(0xc00205bf60, 0xc00205be78, 0x0?, 0x1, 0xc000f1d6e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00205be30 sp=0xc00205bce0 pc=0x48916b +os/exec.(*Cmd).watchCtx(0xc00116e840, 0xc001bd4900) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:757 +0x10f fp=0xc00205bfb0 sp=0xc00205be30 pc=0x656a6f +os/exec.(*Cmd).Start.func4() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0x45 fp=0xc00205bfe0 sp=0xc00205bfb0 pc=0x6565a5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00205bfe8 sp=0xc00205bfe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 609 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0xf05 + +goroutine 661 [syscall]: +syscall.Syscall6(0x0?, 0x0?, 0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/syscall/syscall_linux.go:91 +0x3d fp=0xc001bc0d38 sp=0xc001bc0cb0 pc=0x4d1b1d +os.(*Process).blockUntilWaitable(0xc0009ea8a0) + /home/ec2-user/sdk/go1.21.0/src/os/wait_waitid.go:32 +0x9f fp=0xc001bc0e18 sp=0xc001bc0d38 pc=0x529c5f +os.(*Process).wait(0xc0009ea8a0) + /home/ec2-user/sdk/go1.21.0/src/os/exec_unix.go:22 +0x4b fp=0xc001bc0e98 sp=0xc001bc0e18 pc=0x51e0ab +os.(*Process).Wait(...) + /home/ec2-user/sdk/go1.21.0/src/os/exec.go:134 +os/exec.(*Cmd).Wait(0xc00116e840) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:890 +0xb9 fp=0xc001bc0f60 sp=0xc001bc0e98 pc=0x657279 +github.com/onsi/gomega/gexec.(*Session).monitorForExit(0xc0014cac00, 0x200a3636202d2020?) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:196 +0x3f fp=0xc001bc0fb0 sp=0xc001bc0f60 pc=0x233731f +github.com/onsi/gomega/gexec.Start.func1() + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x45 fp=0xc001bc0fe0 sp=0xc001bc0fb0 pc=0x2336d65 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc0fe8 sp=0xc001bc0fe0 pc=0x4ad421 +created by github.com/onsi/gomega/gexec.Start in goroutine 609 + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x9eb + +goroutine 662 [chan receive]: +runtime.gopark(0x202020200a303231?, 0x20200a343131202d?, 0x20?, 0x20?, 0x3638202d20202020?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015e4e78 sp=0xc0015e4e58 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0015e4ef0 sp=0xc0015e4e78 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x20200a343031202d?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0015e4f18 sp=0xc0015e4ef0 pc=0x440f72 +github.com/rancher/opni/pkg/test.(*Environment).startJetstream.func1() + /home/ec2-user/opni/pkg/test/environment.go:562 +0x69 fp=0xc0015e4f50 sp=0xc0015e4f18 pc=0x24b23e9 +github.com/rancher/opni/pkg/util/waitctx.restrictive.Go.func1() + /home/ec2-user/opni/pkg/util/waitctx/context.go:162 +0x93 fp=0xc0015e4fe0 sp=0xc0015e4f50 pc=0x178df73 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015e4fe8 sp=0xc0015e4fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/util/waitctx.restrictive.Go in goroutine 609 + /home/ec2-user/opni/pkg/util/waitctx/context.go:160 +0xeb + +goroutine 1121 [select]: +runtime.gopark(0xc00129ff40?, 0x3?, 0xb4?, 0x1b?, 0xc00129fd8a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00129fb90 sp=0xc00129fb70 pc=0x4777ae +runtime.selectgo(0xc00129ff40, 0xc00129fd84, 0x5?, 0x0, 0x40ed7c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00129fce0 sp=0xc00129fb90 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).runSync(0xc00070a000) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/admin.go:530 +0x80f fp=0xc00129ffc0 sp=0xc00129fce0 pc=0x396034f +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI.func3() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:90 +0x34 fp=0xc00129ffe0 sp=0xc00129ffc0 pc=0x396f094 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00129ffe8 sp=0xc00129ffe0 pc=0x4ad421 +created by github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI in goroutine 886 + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:90 +0x5cb + +goroutine 1154 [select]: +runtime.gopark(0xc002ec1f40?, 0x3?, 0x0?, 0x1?, 0xc002ec1d2a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ec1b60 sp=0xc002ec1b40 pc=0x4777ae +runtime.selectgo(0xc002ec1f40, 0xc002ec1d24, 0xc002bfaf0b?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ec1cb0 sp=0xc002ec1b60 pc=0x48916b +github.com/rancher/opni/pkg/health.(*Monitor).WatchHealthStatus.func1() + /home/ec2-user/opni/pkg/health/monitor.go:166 +0x68e fp=0xc002ec1fe0 sp=0xc002ec1cb0 pc=0x207082e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ec1fe8 sp=0xc002ec1fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/health.(*Monitor).WatchHealthStatus in goroutine 1136 + /home/ec2-user/opni/pkg/health/monitor.go:150 +0x8da + +goroutine 1085 [select]: +runtime.gopark(0xc002172420?, 0x3?, 0xa0?, 0x9a?, 0xc000f20ada?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d7808 sp=0xc0018d77e8 pc=0x4777ae +runtime.selectgo(0xc002172420, 0xc000f20ad4, 0x0?, 0x0, 0xc00128fef0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0018d7958 sp=0xc0018d7808 pc=0x48916b +reflect.rselect({0xc0018d7b40, 0x3, 0xc00294abd0?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:589 +0x345 fp=0xc0018d7a08 sp=0xc0018d7958 pc=0x4a8f05 +reflect.Select({0xc001702ee0, 0x3, 0x2?}) + /home/ec2-user/sdk/go1.21.0/src/reflect/value.go:3104 +0xb8a fp=0xc0018d7bd0 sp=0xc0018d7a08 pc=0x57c44a +github.com/rancher/opni/pkg/util.WaitAll({0x5b426e0, 0xc0007639a0}, 0xc00128fe70, {0xc0018d7db8, 0x2, 0x2}) + /home/ec2-user/opni/pkg/util/servers.go:51 +0x3ea fp=0xc0018d7d50 sp=0xc0018d7bd0 pc=0x176ae8a +github.com/rancher/opni/pkg/gateway.(*Gateway).ListenAndServe(0xc00135f200, {0x5b426e0, 0xc001fee820}) + /home/ec2-user/opni/pkg/gateway/gateway.go:347 +0x2ae fp=0xc0018d7dd8 sp=0xc0018d7d50 pc=0x22b25ae +github.com/rancher/opni/pkg/test.(*Environment).startGateway.func3(0x4b0571) + /home/ec2-user/opni/pkg/test/environment.go:1709 +0x18c fp=0xc0018d7f88 sp=0xc0018d7dd8 pc=0x24c538c +github.com/rancher/opni/pkg/plugins/hooks.onLoadingCompletedHook.Invoke(0xc001ded3c0, 0xc00158aec0?) + /home/ec2-user/opni/pkg/plugins/hooks/complete.go:10 +0x34 fp=0xc0018d7fa8 sp=0xc0018d7f88 pc=0x2182b94 +github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete.func1() + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x51 fp=0xc0018d7fe0 sp=0xc0018d7fa8 pc=0x2189711 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d7fe8 sp=0xc0018d7fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete in goroutine 609 + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x138 + +goroutine 668 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00205c830 sp=0xc00205c810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00205c868 sp=0xc00205c830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b1a0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00205c888 sp=0xc00205c868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182c720, 0xc001830000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00205c8d0 sp=0xc00205c888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00182c700, {0xc001830000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00205c9c0 sp=0xc00205c8d0 pc=0x50e5a5 +net.(*netFD).Read(0xc00182c700, {0xc001830000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00205ca20 sp=0xc00205c9c0 pc=0x89342b +net.(*conn).Read(0xc001ce24d0, {0xc001830000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00205caa8 sp=0xc00205ca20 pc=0x8b176d +net.(*TCPConn).Read(0xc001ce24d0, {0xc001830000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc00205caf0 sp=0xc00205caa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc000f1b0e0, {0xc000978660, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00205cbb0 sp=0xc00205caf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000f1b0e0}, {0xc000978660, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00205cbf8 sp=0xc00205cbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000978660, 0x9, 0x9}, {0x5b0f0a0, 0xc000f1b0e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00205cc98 sp=0xc00205cbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000978620) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00205cdd0 sp=0xc00205cc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000f94b40, 0xc000f1b140?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc00205cfb0 sp=0xc00205cdd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc00205cfe0 sp=0xc00205cfb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00205cfe8 sp=0xc00205cfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 639 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 669 [select]: +runtime.gopark(0xc001887e70?, 0x2?, 0x0?, 0x0?, 0xc001887df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000ba9c58 sp=0xc000ba9c38 pc=0x4777ae +runtime.selectgo(0xc000ba9e70, 0xc001887df0, 0xc001887e00?, 0x0, 0xc001ce24d0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000ba9da8 sp=0xc000ba9c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001fef860, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc000ba9ea0 sp=0xc000ba9da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001bf39d0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc000ba9f58 sp=0xc000ba9ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc000ba9fe0 sp=0xc000ba9f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000ba9fe8 sp=0xc000ba9fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 639 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 676 [select]: +runtime.gopark(0xc0018d5f70?, 0x4?, 0x58?, 0x5c?, 0xc0018d5dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018d5c00 sp=0xc0018d5be0 pc=0x4777ae +runtime.selectgo(0xc0018d5f70, 0xc0018d5dd0, 0x40a16a0?, 0x0, 0xc00143a750?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0018d5d50 sp=0xc0018d5c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001d65ba0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc0018d5fc0 sp=0xc0018d5d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc0018d5fe0 sp=0xc0018d5fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018d5fe8 sp=0xc0018d5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 674 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 677 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0014897d8 sp=0xc0014897b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001489810 sp=0xc0014897d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b0a8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001489830 sp=0xc001489810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001c9bca0, 0xc0017aa000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001489878 sp=0xc001489830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001c9bc80, {0xc0017aa000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001489968 sp=0xc001489878 pc=0x50e5a5 +net.(*netFD).Read(0xc001c9bc80, {0xc0017aa000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0014899c8 sp=0xc001489968 pc=0x89342b +net.(*conn).Read(0xc001d6f220, {0xc0017aa000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001489a50 sp=0xc0014899c8 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6f220, {0xc0017aa000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc001489a98 sp=0xc001489a50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc000bc2420, {0xc0008812a0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001489b58 sp=0xc001489a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000bc2420}, {0xc0008812a0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001489ba0 sp=0xc001489b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0008812a0, 0x9, 0x9}, {0x5b0f0a0, 0xc000bc2420}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001489c40 sp=0xc001489ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000881260) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001489d78 sp=0xc001489c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001d65ba0, 0x616d656863732422?, 0x3a70747468223a22?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001489ec8 sp=0xc001489d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc000e0c3c0, {0x5b54e00?, 0xc001d65ba0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001489f80 sp=0xc001489ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001489fe0 sp=0xc001489f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001489fe8 sp=0xc001489fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 674 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 671 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0x28?, 0xce?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00205ae18 sp=0xc00205adf8 pc=0x4777ae +runtime.chanrecv(0xc001bd5b60, 0xc00205aed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00205ae90 sp=0xc00205ae18 pc=0x441325 +runtime.chanrecv2(0xc0020ace28?, 0x202d202020200a39?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc00205aeb8 sp=0xc00205ae90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc0020ace00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc00205afc0 sp=0xc00205aeb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc00205afe0 sp=0xc00205afc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00205afe8 sp=0xc00205afe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 609 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 672 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc00200d6a0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f3cf08 sp=0xc000f3cee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc001641cd0, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc000f3cf58 sp=0xc000f3cf08 pc=0x4a9339 +sync.(*Cond).Wait(0xc001641cc0) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc000f3cf88 sp=0xc000f3cf58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc00200d6a0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc000f3cfc0 sp=0xc000f3cf88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc000f3cfe0 sp=0xc000f3cfc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f3cfe8 sp=0xc000f3cfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 609 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 673 [chan receive]: +runtime.gopark(0x36202d202020200a?, 0x202d202020200a36?, 0x31?, 0x31?, 0x2020200a3137202d?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a9f08 sp=0xc0017a9ee8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0017a9f80 sp=0xc0017a9f08 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x200a303231202d20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0017a9fa8 sp=0xc0017a9f80 pc=0x440f72 +github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore.func3() + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:99 +0x4a fp=0xc0017a9fe0 sp=0xc0017a9fa8 pc=0x24a06aa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a9fe8 sp=0xc0017a9fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore in goroutine 609 + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:98 +0x4cb + +goroutine 690 [select]: +runtime.gopark(0xc001bd5e10?, 0x13b0513?, 0x0?, 0x1?, 0xc00205decc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00205dd30 sp=0xc00205dd10 pc=0x4777ae +runtime.selectgo(0xc00205df60, 0xc00205dec8, 0x202020200a353720?, 0x1, 0x20200a3638202d20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00205de80 sp=0xc00205dd30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc000f1b5c0) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:64 +0x318 fp=0xc00205dfc0 sp=0xc00205de80 pc=0x13b0538 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc00205dfe0 sp=0xc00205dfc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00205dfe8 sp=0xc00205dfe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 609 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 730 [select]: +runtime.gopark(0xc0017a6f48?, 0x2?, 0x20?, 0x6f?, 0xc0017a6f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a6d80 sp=0xc0017a6d60 pc=0x4777ae +runtime.selectgo(0xc0017a6f48, 0xc0017a6f00, 0x0?, 0x0, 0x20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017a6ed0 sp=0xc0017a6d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).Recv(0xc0010e4930) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:124 +0xeb fp=0xc0017a6f78 sp=0xc0017a6ed0 pc=0x2154feb +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc000b538b0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc0017a6fc0 sp=0xc0017a6f78 pc=0x215759f +github.com/hashicorp/go-plugin.(*GRPCServer).Init.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x34 fp=0xc0017a6fe0 sp=0xc0017a6fc0 pc=0x215adf4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a6fe8 sp=0xc0017a6fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*GRPCServer).Init in goroutine 729 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x89d + +goroutine 1140 [select]: +runtime.gopark(0xc000a2bf50?, 0x2?, 0xb4?, 0x1b?, 0xc000a2be0c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000a2bc48 sp=0xc000a2bc28 pc=0x4777ae +runtime.selectgo(0xc000a2bf50, 0xc000a2be08, 0xc0018b8620?, 0x0, 0x400?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000a2bd98 sp=0xc000a2bc48 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).watchCortexClusterStatus(0xc00070a000) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/management.go:70 +0x556 fp=0xc000a2bfa0 sp=0xc000a2bd98 pc=0x3963d76 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).useWatchers.func3() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:248 +0x2f fp=0xc000a2bfc0 sp=0xc000a2bfa0 pc=0x3972dcf +github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents.func1() + /home/ec2-user/opni/pkg/management/watcher.go:27 +0x2f fp=0xc000a2bfe0 sp=0xc000a2bfc0 pc=0x23220cf +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a2bfe8 sp=0xc000a2bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents in goroutine 886 + /home/ec2-user/opni/pkg/management/watcher.go:26 +0x54 + +goroutine 1139 [select]: +runtime.gopark(0xc002fa3480?, 0x2?, 0xc0?, 0x58?, 0xc002fa3404?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002fa3270 sp=0xc002fa3250 pc=0x4777ae +runtime.selectgo(0xc002fa3480, 0xc002fa3400, 0xc002fa3510?, 0x0, 0xc002170000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002fa33c0 sp=0xc002fa3270 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002463a40, {0xc001e520d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002fa34b0 sp=0xc002fa33c0 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002463a40, {0xc001e520d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002fa3548 sp=0xc002fa34b0 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001e52060, {0xc001e520d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002fa3590 sp=0xc002fa3548 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001e52060}, {0xc001e520d0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002fa35d8 sp=0xc002fa3590 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0015f8c60, {0xc001e520d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002fa3648 sp=0xc002fa35d8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001e520c0, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002fa3718 sp=0xc002fa3648 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x0?, 0xc0015f8c60, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002fa3808 sp=0xc002fa3718 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001e520c0, {0x7f424347ae80, 0x779c980}, 0x687daa?, {0x0, 0x0}, {0x42d9ca0, 0xc00303cc00}, 0xc002f8a9c0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002fa38c0 sp=0xc002fa3808 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ee3380, {0x42d9ca0?, 0xc00303cc00}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002fa3b20 sp=0xc002fa38c0 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0015f82f4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002fa3b80 sp=0xc002fa3b20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0015f8240, 0xc002fa3d38, 0xc002fa3d08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002fa3c28 sp=0xc002fa3b80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0015f8240, {0x42d9ca0, 0xc00303cc00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002fa3d68 sp=0xc002fa3c28 pc=0x119077e +github.com/rancher/opni/pkg/apis/management/v1.(*managementWatchClusterHealthStatusClient).Recv(0xc000b6f4e0) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:252 +0x66 fp=0xc002fa3da8 sp=0xc002fa3d68 pc=0x177d086 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).watchGlobalClusterHealthStatus(0xc00070a000, {0x5b6f168?, 0xc001a7f950}, 0xc002b0dc20) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/management.go:228 +0x642 fp=0xc002fa3f68 sp=0xc002fa3da8 pc=0x3965de2 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).useWatchers.func2() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:247 +0x1be fp=0xc002fa3fc0 sp=0xc002fa3f68 pc=0x3972fbe +github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents.func1() + /home/ec2-user/opni/pkg/management/watcher.go:27 +0x2f fp=0xc002fa3fe0 sp=0xc002fa3fc0 pc=0x23220cf +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002fa3fe8 sp=0xc002fa3fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents in goroutine 886 + /home/ec2-user/opni/pkg/management/watcher.go:26 +0x54 + +goroutine 733 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0016f1940 sp=0xc0016f1920 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0016f1978 sp=0xc0016f1940 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163afb0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0016f1998 sp=0xc0016f1978 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193a520, 0xc0016f1a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0016f19e0 sp=0xc0016f1998 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00193a500) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0016f1ad0 sp=0xc0016f19e0 pc=0x513f65 +net.(*netFD).accept(0xc00193a500) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0016f1bc8 sp=0xc0016f1ad0 pc=0x8967be +net.(*UnixListener).accept(0xc0010e42d0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc0016f1c08 sp=0xc0016f1bc8 pc=0x8cedfe +net.(*UnixListener).Accept(0xc0010e42d0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc0016f1c78 sp=0xc0016f1c08 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc001ded720) + :1 +0x44 fp=0xc0016f1cb0 sp=0xc0016f1c78 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc00191c5a0, {0x5b2c6a0, 0xc001ded720}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0016f1ee0 sp=0xc0016f1cb0 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCServer).Serve(0xc00135fb00, {0x5b2c6a0, 0xc001ded720}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:153 +0xdd fp=0xc0016f1f98 sp=0xc0016f1ee0 pc=0x215b33d +github.com/hashicorp/go-plugin.Serve.func6() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x62 fp=0xc0016f1fe0 sp=0xc0016f1f98 pc=0x2166b62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0016f1fe8 sp=0xc0016f1fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.Serve in goroutine 729 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x1f67 + +goroutine 691 [chan receive]: +runtime.gopark(0x0?, 0x10?, 0xa0?, 0xf0?, 0xc001620dc4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019f6d10 sp=0xc0019f6cf0 pc=0x4777ae +runtime.chanrecv(0xc0010fe3c0, 0xc0019f6e40, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0019f6d88 sp=0xc0019f6d10 pc=0x441325 +runtime.chanrecv2(0x3b9aca00?, 0x40651e0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0019f6db0 sp=0xc0019f6d88 pc=0x440f92 +github.com/hashicorp/go-plugin.pidWait(0x1860a0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/process.go:20 +0xdb fp=0xc0019f6e68 sp=0xc0019f6db0 pc=0x216033b +github.com/hashicorp/go-plugin.(*Client).reattach.func1(0xc00135f950?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:853 +0xd8 fp=0xc0019f6fb8 sp=0xc0019f6e68 pc=0x21528b8 +github.com/hashicorp/go-plugin.(*Client).reattach.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:862 +0x42 fp=0xc0019f6fe0 sp=0xc0019f6fb8 pc=0x21527a2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019f6fe8 sp=0xc0019f6fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*Client).reattach in goroutine 707 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:846 +0x365 + +goroutine 1057 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc00237c600?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002400f08 sp=0xc002400ee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002722690, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc002400f58 sp=0xc002400f08 pc=0x4a9339 +sync.(*Cond).Wait(0xc002722680) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc002400f88 sp=0xc002400f58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc00237c600) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc002400fc0 sp=0xc002400f88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc002400fe0 sp=0xc002400fc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002400fe8 sp=0xc002400fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 823 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 1442 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ba3d60 sp=0xc002ba3d40 pc=0x4777ae +runtime.chanrecv(0xc001f03080, 0xc002ba3e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002ba3dd8 sp=0xc002ba3d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002ba3e00 sp=0xc002ba3dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002ba3fe0 sp=0xc002ba3e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ba3fe8 sp=0xc002ba3fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 712 [select]: +runtime.gopark(0xc0020e7da8?, 0x2?, 0x60?, 0x78?, 0xc0020e7a94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020e7818 sp=0xc0020e77f8 pc=0x4777ae +runtime.selectgo(0xc0020e7da8, 0xc0020e7a90, 0x0?, 0x0, 0x1860a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020e7968 sp=0xc0020e7818 pc=0x48916b +github.com/hashicorp/go-plugin.Serve(0xc001305d40) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:474 +0x2091 fp=0xc0020e7fc0 sp=0xc0020e7968 pc=0x21665b1 +github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins.func4() + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x34 fp=0xc0020e7fe0 sp=0xc0020e7fc0 pc=0x24cd234 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020e7fe8 sp=0xc0020e7fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins in goroutine 609 + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x76b + +goroutine 692 [select]: +runtime.gopark(0xc00198df70?, 0x2?, 0xb4?, 0x1b?, 0xc00198debc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00198dd20 sp=0xc00198dd00 pc=0x4777ae +runtime.selectgo(0xc00198df70, 0xc00198deb8, 0xc001fef950?, 0x0, 0x202020200a353720?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00198de70 sp=0xc00198dd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001f0f480, {0x5b426e0, 0xc001fefb80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc00198dfa0 sp=0xc00198de70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc00198dfe0 sp=0xc00198dfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00198dfe8 sp=0xc00198dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 693 [select]: +runtime.gopark(0xc0016f5f70?, 0x2?, 0xb4?, 0x1b?, 0xc0016f5ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0016f5d20 sp=0xc0016f5d00 pc=0x4777ae +runtime.selectgo(0xc0016f5f70, 0xc0016f5eb8, 0x657473756c437265?, 0x0, 0x223a226570797422?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0016f5e70 sp=0xc0016f5d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001f0f4e0, {0x5b426e0, 0xc001fefbd0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0016f5fa0 sp=0xc0016f5e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0016f5fe0 sp=0xc0016f5fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0016f5fe8 sp=0xc0016f5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 713 [select]: +runtime.gopark(0xc001967f48?, 0x2?, 0x20?, 0x7f?, 0xc001967f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001967d80 sp=0xc001967d60 pc=0x4777ae +runtime.selectgo(0xc001967f48, 0xc001967f00, 0x0?, 0x0, 0x20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001967ed0 sp=0xc001967d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).Recv(0xc001267050) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:124 +0xeb fp=0xc001967f78 sp=0xc001967ed0 pc=0x2154feb +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc00106e460) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc001967fc0 sp=0xc001967f78 pc=0x215759f +github.com/hashicorp/go-plugin.(*GRPCServer).Init.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x34 fp=0xc001967fe0 sp=0xc001967fc0 pc=0x215adf4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001967fe8 sp=0xc001967fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*GRPCServer).Init in goroutine 712 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x89d + +goroutine 1138 [select]: +runtime.gopark(0xc000f77580?, 0x2?, 0xc0?, 0x72?, 0xc000f77504?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f77370 sp=0xc000f77350 pc=0x4777ae +runtime.selectgo(0xc000f77580, 0xc000f77500, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f774c0 sp=0xc000f77370 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002462730, {0xc002c01ae0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc000f775b0 sp=0xc000f774c0 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002462730, {0xc002c01ae0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc000f77648 sp=0xc000f775b0 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c01aa0, {0xc002c01ae0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc000f77690 sp=0xc000f77648 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c01aa0}, {0xc002c01ae0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000f776d8 sp=0xc000f77690 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0003bfb00, {0xc002c01ae0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc000f77748 sp=0xc000f776d8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c01ad0, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc000f77818 sp=0xc000f77748 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc0003bfb00, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc000f77908 sp=0xc000f77818 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c01ad0, {0x7f424347ae80, 0x779c980}, 0xc000f77a30?, {0x0, 0x0}, {0x432c3a0, 0xc002570980}, 0x4aead9?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc000f779c0 sp=0xc000f77908 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ee31e0, {0x432c3a0?, 0xc002570980}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc000f77c20 sp=0xc000f779c0 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc002b2bdf4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc000f77c80 sp=0xc000f77c20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc002b2bd40, 0xc000f77e38, 0xc000f77e08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc000f77d28 sp=0xc000f77c80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc002b2bd40, {0x432c3a0, 0xc002570980}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc000f77e68 sp=0xc000f77d28 pc=0x119077e +github.com/rancher/opni/pkg/apis/management/v1.(*managementWatchClustersClient).Recv(0xc000b6ee30) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:184 +0x66 fp=0xc000f77ea8 sp=0xc000f77e68 pc=0x177c726 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).watchGlobalCluster(0xc00070a000, {0x5b6f168, 0xc001a7f950}, 0xc001173950?) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/management.go:124 +0x1de fp=0xc000f77f70 sp=0xc000f77ea8 pc=0x396477e +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).useWatchers.func1() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:246 +0x55 fp=0xc000f77fc0 sp=0xc000f77f70 pc=0x3973035 +github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents.func1() + /home/ec2-user/opni/pkg/management/watcher.go:27 +0x2f fp=0xc000f77fe0 sp=0xc000f77fc0 pc=0x23220cf +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f77fe8 sp=0xc000f77fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.(*internalWatcher).WatchEvents in goroutine 886 + /home/ec2-user/opni/pkg/management/watcher.go:26 +0x54 + +goroutine 1159 [select]: +runtime.gopark(0xc001ef4ee0?, 0x2?, 0xb4?, 0x1b?, 0xc001ef4e8c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef4d00 sp=0xc001ef4ce0 pc=0x4777ae +runtime.selectgo(0xc001ef4ee0, 0xc001ef4e88, 0xc001ef4f90?, 0x0, 0xc00290d5f0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ef4e50 sp=0xc001ef4d00 pc=0x48916b +github.com/rancher/opni/pkg/util.ServeHandler({0x5b426e0, 0xc00139cf00}, {0x5b13c00?, 0xc0012c49c0}, {0x5b2a5b0?, 0xc0028fd4d0}) + /home/ec2-user/opni/pkg/util/servers.go:21 +0x26a fp=0xc001ef4f10 sp=0xc001ef4e50 pc=0x176a94a +github.com/rancher/opni/pkg/gateway.(*GatewayHTTPServer).ListenAndServe.func1() + /home/ec2-user/opni/pkg/gateway/http.go:178 +0x199 fp=0xc001ef4fa0 sp=0xc001ef4f10 pc=0x22b8879 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc001ef4fe0 sp=0xc001ef4fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef4fe8 sp=0xc001ef4fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1087 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 742 [select]: +runtime.gopark(0xc001985fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001985f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001985de8 sp=0xc001985dc8 pc=0x4777ae +runtime.selectgo(0xc001985fb0, 0xc001985f70, 0xc001985fd0?, 0x0, 0xc0018b8150?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001985f38 sp=0xc001985de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001985fe0 sp=0xc001985f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001985fe8 sp=0xc001985fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 695 [select]: +runtime.gopark(0xc002056f48?, 0x2?, 0x20?, 0xa8?, 0xc002056f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002056d80 sp=0xc002056d60 pc=0x4777ae +runtime.selectgo(0xc002056f48, 0xc002056f00, 0xc002056f08?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002056ed0 sp=0xc002056d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).Recv(0xc00141d900) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:234 +0xeb fp=0xc002056f78 sp=0xc002056ed0 pc=0x2155d2b +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc001fefdb0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc002056fc0 sp=0xc002056f78 pc=0x215759f +github.com/hashicorp/go-plugin.newGRPCClient.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x34 fp=0xc002056fe0 sp=0xc002056fc0 pc=0x21599b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002056fe8 sp=0xc002056fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 707 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x5b7 + +goroutine 696 [select]: +runtime.gopark(0xc001751748?, 0x2?, 0xd0?, 0x15?, 0xc0017516fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001751578 sp=0xc001751558 pc=0x4777ae +runtime.selectgo(0xc001751748, 0xc0017516f8, 0xc001262460?, 0x0, 0xc00182cc18?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017516c8 sp=0xc001751578 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00165c000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001751778 sp=0xc0017516c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001589ee0, {0x43dd860?, 0xc000bdc960}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0017519d8 sp=0xc001751778 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001f19f14?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001751a38 sp=0xc0017519d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001f19e60, 0xc001751bf0, 0xc001751bc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc001751ae0 sp=0xc001751a38 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f19e60, {0x43dd860, 0xc000bdc960}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001751c20 sp=0xc001751ae0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc00134b7c0, {0x43dd860, 0xc000bdc960}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc001751de8 sp=0xc001751c20 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamClient).Recv(0xc00143a8d0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:132 +0x66 fp=0xc001751e28 sp=0xc001751de8 pc=0x2102606 +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream(0xc00141d900) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:197 +0x2e8 fp=0xc001751fc0 sp=0xc001751e28 pc=0x2155468 +github.com/hashicorp/go-plugin.newGRPCClient.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x34 fp=0xc001751fe0 sp=0xc001751fc0 pc=0x2159954 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001751fe8 sp=0xc001751fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 707 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x62e + +goroutine 716 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001995940 sp=0xc001995920 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001995978 sp=0xc001995940 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163aad8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001995998 sp=0xc001995978 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014c9620, 0xc001995a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0019959e0 sp=0xc001995998 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0014c9600) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001995ad0 sp=0xc0019959e0 pc=0x513f65 +net.(*netFD).accept(0xc0014c9600) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001995bc8 sp=0xc001995ad0 pc=0x8967be +net.(*UnixListener).accept(0xc0012669f0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001995c08 sp=0xc001995bc8 pc=0x8cedfe +net.(*UnixListener).Accept(0xc0012669f0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001995c78 sp=0xc001995c08 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc000ee67a0) + :1 +0x44 fp=0xc001995cb0 sp=0xc001995c78 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc0004ef0e0, {0x5b2c6a0, 0xc000ee67a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001995ee0 sp=0xc001995cb0 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCServer).Serve(0xc001173a70, {0x5b2c6a0, 0xc000ee67a0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:153 +0xdd fp=0xc001995f98 sp=0xc001995ee0 pc=0x215b33d +github.com/hashicorp/go-plugin.Serve.func6() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x62 fp=0xc001995fe0 sp=0xc001995f98 pc=0x2166b62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001995fe8 sp=0xc001995fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.Serve in goroutine 712 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x1f67 + +goroutine 1072 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029bd9e0 sp=0xc0029bd9c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0029bda18 sp=0xc0029bd9e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163a8e8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0029bda38 sp=0xc0029bda18 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001508320, 0x4b0501?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0029bda80 sp=0xc0029bda38 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc001508300) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0029bdb70 sp=0xc0029bda80 pc=0x513f65 +net.(*netFD).accept(0xc001508300) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0029bdc68 sp=0xc0029bdb70 pc=0x8967be +net.(*TCPListener).accept(0xc001331240) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc0029bdcb0 sp=0xc0029bdc68 pc=0x8c383e +net.(*TCPListener).Accept(0xc001331240) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc0029bdd20 sp=0xc0029bdcb0 pc=0x8c1ba5 +google.golang.org/grpc.(*Server).Serve(0xc00191c1e0, {0x5b2a6d0, 0xc001331240}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0029bdf50 sp=0xc0029bdd20 pc=0x117789e +github.com/rancher/opni/pkg/management.(*Server).listenAndServeGrpc.func1() + /home/ec2-user/opni/pkg/management/server.go:213 +0x5e fp=0xc0029bdfa0 sp=0xc0029bdf50 pc=0x231e69e +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0029bdfe0 sp=0xc0029bdfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029bdfe8 sp=0xc0029bdfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1143 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 743 [select]: +runtime.gopark(0xc001964fb0?, 0x2?, 0xb0?, 0x4d?, 0xc001964ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001964d58 sp=0xc001964d38 pc=0x4777ae +runtime.selectgo(0xc001964fb0, 0xc001964ee0, 0xc001964f98?, 0x0, 0xc0010e58f0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001964ea8 sp=0xc001964d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001964fe0 sp=0xc001964ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001964fe8 sp=0xc001964fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 718 [chan receive]: +runtime.gopark(0x0?, 0x10?, 0xa0?, 0xf0?, 0xc000953664?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002058d10 sp=0xc002058cf0 pc=0x4777ae +runtime.chanrecv(0xc000bc3260, 0xc002058e40, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002058d88 sp=0xc002058d10 pc=0x441325 +runtime.chanrecv2(0x3b9aca00?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc002058db0 sp=0xc002058d88 pc=0x440f92 +github.com/hashicorp/go-plugin.pidWait(0x1860a0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/process.go:20 +0xdb fp=0xc002058e68 sp=0xc002058db0 pc=0x216033b +github.com/hashicorp/go-plugin.(*Client).reattach.func1(0xc000b3d0b0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:853 +0xd8 fp=0xc002058fb8 sp=0xc002058e68 pc=0x21528b8 +github.com/hashicorp/go-plugin.(*Client).reattach.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:862 +0x42 fp=0xc002058fe0 sp=0xc002058fb8 pc=0x21527a2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002058fe8 sp=0xc002058fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*Client).reattach in goroutine 717 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:846 +0x365 + +goroutine 740 [IO wait]: +runtime.gopark(0xc001988fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001988f98 sp=0xc001988f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001988fd0 sp=0xc001988f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163a600, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001988ff0 sp=0xc001988fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193a6a0, 0xc0026bd800?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001989038 sp=0xc001988ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00193a680, {0xc0026bd800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001989128 sp=0xc001989038 pc=0x50e5a5 +net.(*netFD).Read(0xc00193a680, {0xc0026bd800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001989188 sp=0xc001989128 pc=0x89342b +net.(*conn).Read(0xc000991348, {0xc0026bd800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001989210 sp=0xc001989188 pc=0x8b176d +net.(*UnixConn).Read(0xc000991348, {0xc0026bd800, 0x4800, 0x4800}) + :1 +0x54 fp=0xc001989258 sp=0xc001989210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc002e63f68, {0xc0026bd800, 0x4800, 0x4800}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0019892b8 sp=0xc001989258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000379ea8, {0x5b0f6c0, 0xc002e63f68}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001989320 sp=0xc0019892b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000379c00, {0x5b0f840?, 0xc000991348}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001989390 sp=0xc001989320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000379c00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0019899c0 sp=0xc001989390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000379c00, {0xc0018c2000, 0x8000, 0xc001989af8?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001989aa8 sp=0xc0019899c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc000ee7be0, {0xc0018c2000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001989af0 sp=0xc001989aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc00127b740, {0xc0011d58c0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001989bb0 sp=0xc001989af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00127b740}, {0xc0011d58c0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001989bf8 sp=0xc001989bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d58c0, 0x9, 0x9}, {0x5b0f0a0, 0xc00127b740}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001989c98 sp=0xc001989bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d5880) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001989dd0 sp=0xc001989c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001285b00, 0xc00189f180?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc001989fb0 sp=0xc001989dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc001989fe0 sp=0xc001989fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001989fe8 sp=0xc001989fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 694 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 704 [select]: +runtime.gopark(0xc001993310?, 0x3?, 0x40?, 0x31?, 0xc001993292?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019930f0 sp=0xc0019930d0 pc=0x4777ae +runtime.selectgo(0xc001993310, 0xc00199328c, 0x46216a2?, 0x0, 0x4?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001993240 sp=0xc0019930f0 pc=0x48916b +github.com/hashicorp/go-plugin.(*grpcStdioServer).StreamStdio(0xc00158b400, 0x42219c0?, {0x5b4ebb0, 0xc00114cec0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:61 +0x1e5 fp=0xc001993350 sp=0xc001993240 pc=0x215bbc5 +github.com/hashicorp/go-plugin/internal/plugin._GRPCStdio_StreamStdio_Handler({0x409e580?, 0xc00158b400}, {0x5b4a6f0, 0xc001f0f9e0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:185 +0xf7 fp=0xc0019933b0 sp=0xc001993350 pc=0x2104077 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x409e580, 0xc00158b400}, {0x5b4ab30, 0xc001c0a3f0}, 0xc0001a1818?, 0x47efca8) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001993400 sp=0xc0019933b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x409e580, 0xc00158b400}, {0x5b4ab30, 0xc001c0a3f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001993490 sp=0xc001993400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x409e580, 0xc00158b400}, {0x5b492f8, 0xc001da41e0}, 0xc0001a1818, 0xc00141dec0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0019937d0 sp=0xc001993490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x409e580, 0xc00158b400}, {0x5b492f8, 0xc001da41e0}, 0x3f252a0?, 0xc00114cdf0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001993848 sp=0xc0019937d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000853200, 0xc0010e4ba0, 0x770cba0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001993db0 sp=0xc001993848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000853200, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001993f28 sp=0xc001993db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001993fe0 sp=0xc001993f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001993fe8 sp=0xc001993fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 719 [select]: +runtime.gopark(0xc001f45f70?, 0x2?, 0xb4?, 0x1b?, 0xc001f45ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f45d20 sp=0xc001f45d00 pc=0x4777ae +runtime.selectgo(0xc001f45f70, 0xc001f45eb8, 0x736c61663a227365?, 0x0, 0x7463656a626f223a?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f45e70 sp=0xc001f45d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000ee7580, {0x5b426e0, 0xc00106e910}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001f45fa0 sp=0xc001f45e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001f45fe0 sp=0xc001f45fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f45fe8 sp=0xc001f45fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 720 [select]: +runtime.gopark(0xc001de3f70?, 0x2?, 0xb4?, 0x1b?, 0xc001de3ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001de3d20 sp=0xc001de3d00 pc=0x4777ae +runtime.selectgo(0xc001de3f70, 0xc001de3eb8, 0x63732f34302d7466?, 0x0, 0x23223a2266657224?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001de3e70 sp=0xc001de3d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000ee7600, {0x5b426e0, 0xc00106e960}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001de3fa0 sp=0xc001de3e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001de3fe0 sp=0xc001de3fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001de3fe8 sp=0xc001de3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 699 [select]: +runtime.gopark(0xc0017a8f48?, 0x2?, 0x50?, 0x8d?, 0xc0017a8f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a8d80 sp=0xc0017a8d60 pc=0x4777ae +runtime.selectgo(0xc0017a8f48, 0xc0017a8f00, 0x8000?, 0x0, 0xc001266cf0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017a8ed0 sp=0xc0017a8d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).Recv(0xc00141da40) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:234 +0xeb fp=0xc0017a8f78 sp=0xc0017a8ed0 pc=0x2155d2b +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc0011f24b0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc0017a8fc0 sp=0xc0017a8f78 pc=0x215759f +github.com/hashicorp/go-plugin.newGRPCClient.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x34 fp=0xc0017a8fe0 sp=0xc0017a8fc0 pc=0x21599b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a8fe8 sp=0xc0017a8fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 717 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x5b7 + +goroutine 687 [select]: +runtime.gopark(0xc0017d2fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0017d2f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d2de8 sp=0xc0017d2dc8 pc=0x4777ae +runtime.selectgo(0xc0017d2fb0, 0xc0017d2f70, 0x1179540?, 0x0, 0xc0017d04e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017d2f38 sp=0xc0017d2de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0017d2fe0 sp=0xc0017d2f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d2fe8 sp=0xc0017d2fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 700 [select]: +runtime.gopark(0xc00197b748?, 0x2?, 0xd0?, 0xb5?, 0xc00197b6fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00197b578 sp=0xc00197b558 pc=0x4777ae +runtime.selectgo(0xc00197b748, 0xc00197b6f8, 0xc0012626a0?, 0x0, 0xc00182d118?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00197b6c8 sp=0xc00197b578 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0015f99e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc00197b778 sp=0xc00197b6c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00184a340, {0x43dd860?, 0xc00128c730}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc00197b9d8 sp=0xc00197b778 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000852534?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc00197ba38 sp=0xc00197b9d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc000852480, 0xc00197bbf0, 0xc00197bbc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc00197bae0 sp=0xc00197ba38 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc000852480, {0x43dd860, 0xc00128c730}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc00197bc20 sp=0xc00197bae0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0011615c0, {0x43dd860, 0xc00128c730}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc00197bde8 sp=0xc00197bc20 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamClient).Recv(0xc00158ba70) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:132 +0x66 fp=0xc00197be28 sp=0xc00197bde8 pc=0x2102606 +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream(0xc00141da40) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:197 +0x2e8 fp=0xc00197bfc0 sp=0xc00197be28 pc=0x2155468 +github.com/hashicorp/go-plugin.newGRPCClient.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x34 fp=0xc00197bfe0 sp=0xc00197bfc0 pc=0x2159954 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00197bfe8 sp=0xc00197bfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 717 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x62e + +goroutine 688 [select]: +runtime.gopark(0xc0017a2fb0?, 0x2?, 0xb0?, 0x2d?, 0xc0017a2ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a2d58 sp=0xc0017a2d38 pc=0x4777ae +runtime.selectgo(0xc0017a2fb0, 0xc0017a2ee0, 0xc0017a2f98?, 0x0, 0xc0012b6030?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017a2ea8 sp=0xc0017a2d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0017a2fe0 sp=0xc0017a2ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a2fe8 sp=0xc0017a2fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 736 [IO wait]: +runtime.gopark(0xc001f46fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f46f98 sp=0xc001f46f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001f46fd0 sp=0xc001f46f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163c3f0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001f46ff0 sp=0xc001f46fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0014c9a20, 0xc00298a000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001f47038 sp=0xc001f46ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0014c9a00, {0xc00298a000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001f47128 sp=0xc001f47038 pc=0x50e5a5 +net.(*netFD).Read(0xc0014c9a00, {0xc00298a000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001f47188 sp=0xc001f47128 pc=0x89342b +net.(*conn).Read(0xc000b79a60, {0xc00298a000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001f47210 sp=0xc001f47188 pc=0x8b176d +net.(*UnixConn).Read(0xc000b79a60, {0xc00298a000, 0xa000, 0xa000}) + :1 +0x54 fp=0xc001f47258 sp=0xc001f47210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0032a2a50, {0xc00298a000, 0xa000, 0xa000}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001f472b8 sp=0xc001f47258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00189f428, {0x5b0f6c0, 0xc0032a2a50}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001f47320 sp=0xc001f472b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00189f180, {0x5b0f840?, 0xc000b79a60}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001f47390 sp=0xc001f47320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00189f180, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001f479c0 sp=0xc001f47390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00189f180, {0xc0019e8000, 0x8000, 0xc003300370?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001f47aa8 sp=0xc001f479c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001dedc00, {0xc0019e8000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001f47af0 sp=0xc001f47aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc001db0f60, {0xc001938820, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001f47bb0 sp=0xc001f47af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001db0f60}, {0xc001938820, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001f47bf8 sp=0xc001f47bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001938820, 0x9, 0x9}, {0x5b0f0a0, 0xc001db0f60}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001f47c98 sp=0xc001f47bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0019387e0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001f47dd0 sp=0xc001f47c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001c6c480, 0xc0004ef0e0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc001f47fb0 sp=0xc001f47dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc001f47fe0 sp=0xc001f47fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f47fe8 sp=0xc001f47fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 721 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 741 [select]: +runtime.gopark(0xc001af1e70?, 0x2?, 0x0?, 0x0?, 0xc001af1df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001af1c58 sp=0xc001af1c38 pc=0x4777ae +runtime.selectgo(0xc001af1e70, 0xc001af1df0, 0xc001af1e00?, 0x0, 0xc000379c00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001af1da8 sp=0xc001af1c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0007c00f0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001af1ea0 sp=0xc001af1da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0018b8150) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001af1f58 sp=0xc001af1ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc001af1fe0 sp=0xc001af1f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001af1fe8 sp=0xc001af1fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 694 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 701 [select]: +runtime.gopark(0xc00264de48?, 0x2?, 0x0?, 0x0?, 0xc00264ddcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00264dc30 sp=0xc00264dc10 pc=0x4777ae +runtime.selectgo(0xc00264de48, 0xc00264ddc8, 0xc00264ddd8?, 0x0, 0xc001520380?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00264dd80 sp=0xc00264dc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0011f3220, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00264de78 sp=0xc00264dd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0020127e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00264df30 sp=0xc00264de78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc00264dfe0 sp=0xc00264df30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00264dfe8 sp=0xc00264dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 697 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 702 [select]: +runtime.gopark(0xc001966f70?, 0x4?, 0xc0?, 0x2?, 0xc001966dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001966c00 sp=0xc001966be0 pc=0x4777ae +runtime.selectgo(0xc001966f70, 0xc001966dd0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001966d50 sp=0xc001966c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00183b520) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001966fc0 sp=0xc001966d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001966fe0 sp=0xc001966fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001966fe8 sp=0xc001966fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 697 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 703 [IO wait]: +runtime.gopark(0xc001990f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001990f40 sp=0xc001990f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001990f78 sp=0xc001990f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163a508, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001990f98 sp=0xc001990f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182cd20, 0xc0011a9000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001990fe0 sp=0xc001990f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00182cd00, {0xc0011a9000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0019910d0 sp=0xc001990fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc00182cd00, {0xc0011a9000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001991130 sp=0xc0019910d0 pc=0x89342b +net.(*conn).Read(0xc001ce2600, {0xc0011a9000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0019911b8 sp=0xc001991130 pc=0x8b176d +net.(*UnixConn).Read(0xc001ce2600, {0xc0011a9000, 0x800, 0x800}) + :1 +0x54 fp=0xc001991200 sp=0xc0019911b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc002e63f50, {0xc0011a9000, 0x800, 0x800}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001991260 sp=0xc001991200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc001520628, {0x5b0f6c0, 0xc002e63f50}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0019912c8 sp=0xc001991260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001520380, {0x5b0f840?, 0xc001ce2600}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001991338 sp=0xc0019912c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001520380, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001991968 sp=0xc001991338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001520380, {0xc0019a8000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001991a50 sp=0xc001991968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001f0f900, {0xc0019a8000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001991a98 sp=0xc001991a50 pc=0x10969ed +bufio.(*Reader).Read(0xc0014828a0, {0xc000978740, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001991b58 sp=0xc001991a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0014828a0}, {0xc000978740, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001991ba0 sp=0xc001991b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000978740, 0x9, 0x9}, {0x5b0f0a0, 0xc0014828a0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001991c40 sp=0xc001991ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000978700) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001991d78 sp=0xc001991c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00183b520, 0x78?, 0x1766418d00000000?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001991ec8 sp=0xc001991d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c5a0, {0x5b54e00?, 0xc00183b520}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001991f80 sp=0xc001991ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001991fe0 sp=0xc001991f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001991fe8 sp=0xc001991fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 697 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 744 [chan receive]: +runtime.gopark(0xc001981d80?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001981d60 sp=0xc001981d40 pc=0x4777ae +runtime.chanrecv(0xc001713800, 0xc001981e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001981dd8 sp=0xc001981d60 pc=0x441325 +runtime.chanrecv1(0x1?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001981e00 sp=0xc001981dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001981fe0 sp=0xc001981e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001981fe8 sp=0xc001981fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 680 [select]: +runtime.gopark(0xc001960fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001960f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001960de8 sp=0xc001960dc8 pc=0x4777ae +runtime.selectgo(0xc001960fb0, 0xc001960f70, 0x1deeab63?, 0x0, 0xc0012669f0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001960f38 sp=0xc001960de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001960fe0 sp=0xc001960f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001960fe8 sp=0xc001960fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 696 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 681 [select]: +runtime.gopark(0xc0017d9fb0?, 0x2?, 0xb0?, 0x9d?, 0xc0017d9ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d9d58 sp=0xc0017d9d38 pc=0x4777ae +runtime.selectgo(0xc0017d9fb0, 0xc0017d9ee0, 0xa3736202d202020?, 0x0, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017d9ea8 sp=0xc0017d9d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0017d9fe0 sp=0xc0017d9ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d9fe8 sp=0xc0017d9fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 696 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 682 [chan receive]: +runtime.gopark(0x3332343637313138?, 0x7d2265686361632f?, 0x2c?, 0x22?, 0x223a22656e69676e?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001961d60 sp=0xc001961d40 pc=0x4777ae +runtime.chanrecv(0xc000f27f80, 0xc001961e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001961dd8 sp=0xc001961d60 pc=0x441325 +runtime.chanrecv1(0x706d742f223a2268?, 0x65742d696e706f2f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001961e00 sp=0xc001961dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001961fe0 sp=0xc001961e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001961fe8 sp=0xc001961fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 696 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 683 [select]: +runtime.gopark(0xc00198bfa0?, 0x3?, 0xf0?, 0xbd?, 0xc00198bf3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00198bda8 sp=0xc00198bd88 pc=0x4777ae +runtime.selectgo(0xc00198bfa0, 0xc00198bf34, 0xc001962f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00198bef8 sp=0xc00198bda8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:184 +0x1b3 fp=0xc00198bfe0 sp=0xc00198bef8 pc=0x21557f3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00198bfe8 sp=0xc00198bfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream in goroutine 696 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:182 +0x24a + +goroutine 745 [select]: +runtime.gopark(0xc001a09790?, 0x2?, 0x18?, 0x96?, 0xc001a09744?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a095c0 sp=0xc001a095a0 pc=0x4777ae +runtime.selectgo(0xc001a09790, 0xc001a09740, 0xa10127972746e45?, 0x0, 0x1279656b03520928?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a09710 sp=0xc001a095c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00050e000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001a097c0 sp=0xc001a09710 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001589e10, {0x43c4ea0?, 0xc0012b4500}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc001a09a20 sp=0xc001a097c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001f19df4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001a09a80 sp=0xc001a09a20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001f19d40, 0xc0017d6c38, 0xc0017d6c08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc001a09b28 sp=0xc001a09a80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f19d40, {0x43c4ea0, 0xc0012b4500}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001a09c68 sp=0xc001a09b28 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0012b4280, {0x43c4ea0, 0xc0012b4500}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc001a09e30 sp=0xc001a09c68 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCStdioStreamStdioClient).Recv(0xc001511540) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:160 +0x66 fp=0xc001a09e70 sp=0xc001a09e30 pc=0x2103f26 +github.com/hashicorp/go-plugin.(*grpcStdioClient).Run(0xc000ee7de0, {0x5b0ee60, 0x779c980}, {0x5b0eb80, 0xc00007c038}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:135 +0x11c fp=0xc001a09f80 sp=0xc001a09e70 pc=0x215c15c +github.com/hashicorp/go-plugin.newGRPCClient.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x6c fp=0xc001a09fe0 sp=0xc001a09f80 pc=0x21598ec +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a09fe8 sp=0xc001a09fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 707 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x84f + +goroutine 737 [select]: +runtime.gopark(0xc00256be70?, 0x2?, 0x0?, 0x0?, 0xc00256bdf4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00256bc58 sp=0xc00256bc38 pc=0x4777ae +runtime.selectgo(0xc00256be70, 0xc00256bdf0, 0xc00202ddc0?, 0x0, 0xc00189f180?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00256bda8 sp=0xc00256bc58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00128c5f0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00256bea0 sp=0xc00256bda8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00202ddc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00256bf58 sp=0xc00256bea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc00256bfe0 sp=0xc00256bf58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00256bfe8 sp=0xc00256bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 721 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 746 [select]: +runtime.gopark(0xc001986fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001986f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001986de8 sp=0xc001986dc8 pc=0x4777ae +runtime.selectgo(0xc001986fb0, 0xc001986f70, 0x6b4344646b514e52?, 0x0, 0x6d65356355597a5a?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001986f38 sp=0xc001986de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001986fe0 sp=0xc001986f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001986fe8 sp=0xc001986fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 747 [select]: +runtime.gopark(0xc0017d5fb0?, 0x2?, 0xb0?, 0x5d?, 0xc0017d5ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d5d58 sp=0xc0017d5d38 pc=0x4777ae +runtime.selectgo(0xc0017d5fb0, 0xc0017d5ee0, 0x203a737365726464?, 0x0, 0xa31313635343a74?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017d5ea8 sp=0xc0017d5d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0017d5fe0 sp=0xc0017d5ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d5fe8 sp=0xc0017d5fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 748 [chan receive]: +runtime.gopark(0xc001963d80?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001963d60 sp=0xc001963d40 pc=0x4777ae +runtime.chanrecv(0xc001713ce0, 0xc001963e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001963dd8 sp=0xc001963d60 pc=0x441325 +runtime.chanrecv1(0x1?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001963e00 sp=0xc001963dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001963fe0 sp=0xc001963e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001963fe8 sp=0xc001963fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 705 [select]: +runtime.gopark(0xc001e16a68?, 0x2?, 0x0?, 0x0?, 0xc001e16a14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e16880 sp=0xc001e16860 pc=0x4777ae +runtime.selectgo(0xc001e16a68, 0xc001e16a10, 0x7f429a447100?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e169d0 sp=0xc001e16880 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001434000, {0xc001aa6700, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001e16a98 sp=0xc001e169d0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001434000, {0xc001aa6700, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001e16b30 sp=0xc001e16a98 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001473ef0, {0xc001aa6700, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001e16b78 sp=0xc001e16b30 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001473ef0}, {0xc001aa6700, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001e16bc0 sp=0xc001e16b78 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000853680, {0xc001aa6700, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001e16c30 sp=0xc001e16bc0 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001aa66f0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001e16d00 sp=0xc001e16c30 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc000853680, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001e16df0 sp=0xc001e16d00 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001aa66f0, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43dd860, 0xc001baacd0}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001e16ea8 sp=0xc001e16df0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001da50e0, {0x43dd860?, 0xc001baacd0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001e170e0 sp=0xc001e16ea8 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001aa6870, {0x43dd860, 0xc001baacd0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001e171c8 sp=0xc001e170e0 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001f0fa40, {0x43dd860, 0xc001baacd0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001e17200 sp=0xc001e171c8 pc=0x1fd94b2 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamServer).Recv(0xc00114cfa0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:167 +0x66 fp=0xc001e17240 sp=0xc001e17200 pc=0x2102886 +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream(0xc0010e4930, {0x5b512b8, 0xc00114cfa0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:87 +0x218 fp=0xc001e17370 sp=0xc001e17240 pc=0x2154718 +github.com/hashicorp/go-plugin/internal/plugin._GRPCBroker_StartStream_Handler({0x41cb860?, 0xc0010e4930}, {0x5b4a6f0?, 0xc001f0fa40}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:148 +0xb6 fp=0xc001e173b0 sp=0xc001e17370 pc=0x2102716 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41cb860, 0xc0010e4930}, {0x5b4ab30, 0xc001aa6870}, 0xc0001a18a8?, 0x47efc98) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001e17400 sp=0xc001e173b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41cb860, 0xc0010e4930}, {0x5b4ab30, 0xc001aa6870}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001e17490 sp=0xc001e17400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41cb860, 0xc0010e4930}, {0x5b492f8, 0xc001da50e0}, 0xc0001a18a8, 0xc001bae0c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001e177d0 sp=0xc001e17490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41cb860, 0xc0010e4930}, {0x5b492f8, 0xc001da50e0}, 0x3f252a0?, 0xc00114ced0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001e17848 sp=0xc001e177d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000853680, 0xc0010e49c0, 0x770cb80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001e17db0 sp=0xc001e17848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000853680, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001e17f28 sp=0xc001e17db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001e17fe0 sp=0xc001e17f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e17fe8 sp=0xc001e17fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 684 [select]: +runtime.gopark(0xc002423e48?, 0x2?, 0x0?, 0x0?, 0xc002423dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002423c30 sp=0xc002423c10 pc=0x4777ae +runtime.selectgo(0xc002423e48, 0xc002423dc8, 0xc002423dd8?, 0x0, 0xc001968000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002423d80 sp=0xc002423c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0009d6280, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002423e78 sp=0xc002423d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0017da0e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002423f30 sp=0xc002423e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002423fe0 sp=0xc002423f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002423fe8 sp=0xc002423fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 679 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 685 [select]: +runtime.gopark(0xc001a39f70?, 0x4?, 0x10?, 0x0?, 0xc001a39dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a39c00 sp=0xc001a39be0 pc=0x4777ae +runtime.selectgo(0xc001a39f70, 0xc001a39dd0, 0x40a16a0?, 0x0, 0xc00143a8f0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a39d50 sp=0xc001a39c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0017d04e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001a39fc0 sp=0xc001a39d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001a39fe0 sp=0xc001a39fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a39fe8 sp=0xc001a39fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 679 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 686 [IO wait]: +runtime.gopark(0xc00198ef88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00198ef40 sp=0xc00198ef20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00198ef78 sp=0xc00198ef40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163c2f8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00198ef98 sp=0xc00198ef78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001c9bfa0, 0xc000d98e00?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00198efe0 sp=0xc00198ef98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001c9bf80, {0xc000d98e00, 0x700, 0x700}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00198f0d0 sp=0xc00198efe0 pc=0x50e5a5 +net.(*netFD).Read(0xc001c9bf80, {0xc000d98e00, 0x700, 0x700}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00198f130 sp=0xc00198f0d0 pc=0x89342b +net.(*conn).Read(0xc001d6f240, {0xc000d98e00, 0x700, 0x700}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00198f1b8 sp=0xc00198f130 pc=0x8b176d +net.(*UnixConn).Read(0xc001d6f240, {0xc000d98e00, 0x700, 0x700}) + :1 +0x54 fp=0xc00198f200 sp=0xc00198f1b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0032a2960, {0xc000d98e00, 0x700, 0x700}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc00198f260 sp=0xc00198f200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0019682a8, {0x5b0f6c0, 0xc0032a2960}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc00198f2c8 sp=0xc00198f260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001968000, {0x5b0f840?, 0xc001d6f240}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc00198f338 sp=0xc00198f2c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001968000, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc00198f968 sp=0xc00198f338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001968000, {0xc0017e2000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc00198fa50 sp=0xc00198f968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0007d3060, {0xc0017e2000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc00198fa98 sp=0xc00198fa50 pc=0x10969ed +bufio.(*Reader).Read(0xc0012c8ba0, {0xc000881380, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00198fb58 sp=0xc00198fa98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0012c8ba0}, {0xc000881380, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00198fba0 sp=0xc00198fb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000881380, 0x9, 0x9}, {0x5b0f0a0, 0xc0012c8ba0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00198fc40 sp=0xc00198fba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000881340) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00198fd78 sp=0xc00198fc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0017d04e0, 0x78?, 0x1769ebc60215b33d?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc00198fec8 sp=0xc00198fd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc0004ef0e0, {0x5b54e00?, 0xc0017d04e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc00198ff80 sp=0xc00198fec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc00198ffe0 sp=0xc00198ff80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00198ffe8 sp=0xc00198ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 679 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 749 [select]: +runtime.gopark(0xc000d12890?, 0x2?, 0x4e?, 0x1a?, 0xc000d1283c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d126a8 sp=0xc000d12688 pc=0x4777ae +runtime.selectgo(0xc000d12890, 0xc000d12838, 0x7f429a449e00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000d127f8 sp=0xc000d126a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0007c08c0, {0xc001aa6970, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc000d128c0 sp=0xc000d127f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0007c08c0, {0xc001aa6970, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc000d12958 sp=0xc000d128c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0012b7ad0, {0xc001aa6970, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc000d129a0 sp=0xc000d12958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0012b7ad0}, {0xc001aa6970, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000d129e8 sp=0xc000d129a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008ac360, {0xc001aa6970, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc000d12a58 sp=0xc000d129e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001aa6960, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc000d12b28 sp=0xc000d12a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001baebc0?, 0xc0008ac360, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc000d12c18 sp=0xc000d12b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001aa6960, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0014345f0}, 0xc0010e4480?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc000d12cd0 sp=0xc000d12c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001da5e00, {0x43dcf60?, 0xc0014345f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc000d12f08 sp=0xc000d12cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001aa6ab0, {0x43dcf60, 0xc0014345f0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc000d12ff0 sp=0xc000d12f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001f0faa0, {0x43dcf60, 0xc0014345f0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc000d13028 sp=0xc000d12ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00114d080) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc000d13068 sp=0xc000d13028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00114d080?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc000d130a8 sp=0xc000d13068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00114d090) + :1 +0x46 fp=0xc000d130e0 sp=0xc000d130a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc00114d090}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc000d13320 sp=0xc000d130e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc00114d080}) + :1 +0xaf fp=0xc000d13370 sp=0xc000d13320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc001f0faa0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc000d133b0 sp=0xc000d13370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001aa6ab0}, 0xc0001a1920?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000d13400 sp=0xc000d133b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001aa6ab0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000d13490 sp=0xc000d13400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001da5e00}, 0xc0001a1920, 0xc001bae1c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000d137d0 sp=0xc000d13490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001da5e00}, 0x3f252a0?, 0xc00114cfb0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000d13848 sp=0xc000d137d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc0008ac360, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000d13db0 sp=0xc000d13848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc0008ac360, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000d13f28 sp=0xc000d13db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000d13fe0 sp=0xc000d13f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d13fe8 sp=0xc000d13fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 754 [select]: +runtime.gopark(0xc0017a3fa0?, 0x3?, 0x0?, 0x4e?, 0xc0017a3f3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a3da8 sp=0xc0017a3d88 pc=0x4777ae +runtime.selectgo(0xc0017a3fa0, 0xc0017a3f34, 0xc0017a3f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017a3ef8 sp=0xc0017a3da8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:73 +0x1ad fp=0xc0017a3fe0 sp=0xc0017a3ef8 pc=0x2154aad +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a3fe8 sp=0xc0017a3fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream in goroutine 705 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:71 +0x1d1 + +goroutine 770 [select]: +runtime.gopark(0xc001a07790?, 0x2?, 0x18?, 0x76?, 0xc001a07744?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a075c0 sp=0xc001a075a0 pc=0x4777ae +runtime.selectgo(0xc001a07790, 0xc001a07740, 0x40?, 0x0, 0x40?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a07710 sp=0xc001a075c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00165c480) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001a077c0 sp=0xc001a07710 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00184a270, {0x43c4ea0?, 0xc001baec00}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc001a07a20 sp=0xc001a077c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0008520b4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001a07a80 sp=0xc001a07a20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc000852000, 0xc0017a4c38, 0xc0017a4c08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc001a07b28 sp=0xc001a07a80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc000852000, {0x43c4ea0, 0xc001baec00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001a07c68 sp=0xc001a07b28 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc00134ba80, {0x43c4ea0, 0xc001baec00}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc001a07e30 sp=0xc001a07c68 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCStdioStreamStdioClient).Recv(0xc00143aab0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:160 +0x66 fp=0xc001a07e70 sp=0xc001a07e30 pc=0x2103f26 +github.com/hashicorp/go-plugin.(*grpcStdioClient).Run(0xc0007d3440, {0x5b0ee60, 0x779c980}, {0x5b0eb80, 0xc00007c038}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:135 +0x11c fp=0xc001a07f80 sp=0xc001a07e70 pc=0x215c15c +github.com/hashicorp/go-plugin.newGRPCClient.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x6c fp=0xc001a07fe0 sp=0xc001a07f80 pc=0x21598ec +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a07fe8 sp=0xc001a07fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 717 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x84f + +goroutine 771 [select]: +runtime.gopark(0xc0017d4fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0017d4f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d4de8 sp=0xc0017d4dc8 pc=0x4777ae +runtime.selectgo(0xc0017d4fb0, 0xc0017d4f70, 0x3235202d20202020?, 0x0, 0x202d202020200a38?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017d4f38 sp=0xc0017d4de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0017d4fe0 sp=0xc0017d4f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d4fe8 sp=0xc0017d4fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 772 [select]: +runtime.gopark(0xc001a35fb0?, 0x2?, 0xb0?, 0x5d?, 0xc001a35ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a35d58 sp=0xc001a35d38 pc=0x4777ae +runtime.selectgo(0xc001a35fb0, 0xc001a35ee0, 0x1128d01?, 0x0, 0x1102359?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a35ea8 sp=0xc001a35d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001a35fe0 sp=0xc001a35ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a35fe8 sp=0xc001a35fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 773 [chan receive]: +runtime.gopark(0x5059545f54454752?, 0x10444c4549465f45?, 0x4?, 0x12?, 0x455059545f544547?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a36d60 sp=0xc001a36d40 pc=0x4777ae +runtime.chanrecv(0xc00119a120, 0xc001a36e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001a36dd8 sp=0xc001a36d60 pc=0x441325 +runtime.chanrecv1(0x74704f666f656e4f?, 0x140a5812736e6f69?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001a36e00 sp=0xc001a36dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001a36fe0 sp=0xc001a36e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a36fe8 sp=0xc001a36fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 755 [select]: +runtime.gopark(0xc001b11310?, 0x3?, 0xd9?, 0xea?, 0xc001b11292?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b110f0 sp=0xc001b110d0 pc=0x4777ae +runtime.selectgo(0xc001b11310, 0xc001b1128c, 0x46216a2?, 0x0, 0x4?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b11240 sp=0xc001b110f0 pc=0x48916b +github.com/hashicorp/go-plugin.(*grpcStdioServer).StreamStdio(0xc0015108d0, 0x42219c0?, {0x5b4ebb0, 0xc00114d270}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:61 +0x1e5 fp=0xc001b11350 sp=0xc001b11240 pc=0x215bbc5 +github.com/hashicorp/go-plugin/internal/plugin._GRPCStdio_StreamStdio_Handler({0x409e580?, 0xc0015108d0}, {0x5b4a6f0, 0xc001f0fb40}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:185 +0xf7 fp=0xc001b113b0 sp=0xc001b11350 pc=0x2104077 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x409e580, 0xc0015108d0}, {0x5b4ab30, 0xc001cd81b0}, 0xc0004a6048?, 0x47efca8) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001b11400 sp=0xc001b113b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x409e580, 0xc0015108d0}, {0x5b4ab30, 0xc001cd81b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001b11490 sp=0xc001b11400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x409e580, 0xc0015108d0}, {0x5b492f8, 0xc001da5ef0}, 0xc0004a6048, 0xc001baed80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001b117d0 sp=0xc001b11490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x409e580, 0xc0015108d0}, {0x5b492f8, 0xc001da5ef0}, 0x3f252a0?, 0xc00114d190?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001b11848 sp=0xc001b117d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0008539e0, 0xc001267320, 0x770cba0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001b11db0 sp=0xc001b11848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0008539e0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001b11f28 sp=0xc001b11db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001b11fe0 sp=0xc001b11f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b11fe8 sp=0xc001b11fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 750 [select]: +runtime.gopark(0xc001b33fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b33f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b33de8 sp=0xc001b33dc8 pc=0x4777ae +runtime.selectgo(0xc001b33fb0, 0xc001b33f70, 0xc001713a40?, 0x0, 0xc0012b7110?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b33f38 sp=0xc001b33de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b33fe0 sp=0xc001b33f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b33fe8 sp=0xc001b33fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 751 [select]: +runtime.gopark(0xc0017a4fb0?, 0x2?, 0xb0?, 0x4d?, 0xc0017a4ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a4d58 sp=0xc0017a4d38 pc=0x4777ae +runtime.selectgo(0xc0017a4fb0, 0xc0017a4ee0, 0x5655434a55573374?, 0x0, 0x6a55575255575268?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017a4ea8 sp=0xc0017a4d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0017a4fe0 sp=0xc0017a4ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a4fe8 sp=0xc0017a4fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 752 [chan receive]: +runtime.gopark(0xc001965da8?, 0x1103325?, 0xe0?, 0x87?, 0xc00149cea8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001965d60 sp=0xc001965d40 pc=0x4777ae +runtime.chanrecv(0xc000c9c120, 0xc001965e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001965dd8 sp=0xc001965d60 pc=0x441325 +runtime.chanrecv1(0x1101c91?, 0xc0013219f8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001965e00 sp=0xc001965dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001965fe0 sp=0xc001965e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001965fe8 sp=0xc001965fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 774 [select]: +runtime.gopark(0xc000f0a890?, 0x2?, 0x98?, 0xa7?, 0xc000f0a83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000f0a6a8 sp=0xc000f0a688 pc=0x4777ae +runtime.selectgo(0xc000f0a890, 0xc000f0a838, 0x7f429a44c500?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc000f0a7f8 sp=0xc000f0a6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000b685f0, {0xc001c610c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc000f0a8c0 sp=0xc000f0a7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000b685f0, {0xc001c610c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc000f0a958 sp=0xc000f0a8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001c60e40, {0xc001c610c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc000f0a9a0 sp=0xc000f0a958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001c60e40}, {0xc001c610c0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc000f0a9e8 sp=0xc000f0a9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00165ca20, {0xc001c610c0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc000f0aa58 sp=0xc000f0a9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001c610b0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc000f0ab28 sp=0xc000f0aa58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001e75840?, 0xc00165ca20, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc000f0ac18 sp=0xc000f0ab28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001c610b0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc000b68f00}, 0xc001266cc0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc000f0acd0 sp=0xc000f0ac18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001ec2870, {0x43dcf60?, 0xc000b68f00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc000f0af08 sp=0xc000f0acd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001d42b40, {0x43dcf60, 0xc000b68f00}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc000f0aff0 sp=0xc000f0af08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0007d3680, {0x43dcf60, 0xc000b68f00}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc000f0b028 sp=0xc000f0aff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00143ad60) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc000f0b068 sp=0xc000f0b028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00143ad60?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc000f0b0a8 sp=0xc000f0b068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00143ad70) + :1 +0x46 fp=0xc000f0b0e0 sp=0xc000f0b0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc00143ad70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc000f0b320 sp=0xc000f0b0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc00143ad60}) + :1 +0xaf fp=0xc000f0b370 sp=0xc000f0b320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc0007d3680}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc000f0b3b0 sp=0xc000f0b370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001d42b40}, 0xc0014238f0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc000f0b400 sp=0xc000f0b3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001d42b40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc000f0b490 sp=0xc000f0b400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001ec2870}, 0xc0014238f0, 0xc001e74380) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc000f0b7d0 sp=0xc000f0b490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001ec2870}, 0x3f252a0?, 0xc00143ac90?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc000f0b848 sp=0xc000f0b7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165ca20, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc000f0bdb0 sp=0xc000f0b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165ca20, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc000f0bf28 sp=0xc000f0bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc000f0bfe0 sp=0xc000f0bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f0bfe8 sp=0xc000f0bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 753 [select]: +runtime.gopark(0xc001122890?, 0x2?, 0x4e?, 0x1a?, 0xc00112283c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011226a8 sp=0xc001122688 pc=0x4777ae +runtime.selectgo(0xc001122890, 0xc001122838, 0x7f429a44c200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011227f8 sp=0xc0011226a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0007c1180, {0xc001395270, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0011228c0 sp=0xc0011227f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0007c1180, {0xc001395270, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001122958 sp=0xc0011228c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0013951a0, {0xc001395270, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0011229a0 sp=0xc001122958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0013951a0}, {0xc001395270, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0011229e8 sp=0xc0011229a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000a89c20, {0xc001395270, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001122a58 sp=0xc0011229e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001395260, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001122b28 sp=0xc001122a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0012b59c0?, 0xc000a89c20, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001122c18 sp=0xc001122b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001395260, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0007c14a0}, 0x66?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001122cd0 sp=0xc001122c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc00166bd10, {0x43dcf60?, 0xc0007c14a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001122f08 sp=0xc001122cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001395590, {0x43dcf60, 0xc0007c14a0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001122ff0 sp=0xc001122f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0009d24e0, {0x43dcf60, 0xc0007c14a0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001123028 sp=0xc001122ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc0015119a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001123068 sp=0xc001123028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc0015119a0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0011230a8 sp=0xc001123068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc0015119b0) + :1 +0x46 fp=0xc0011230e0 sp=0xc0011230a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc0015119b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001123320 sp=0xc0011230e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc0015119a0}) + :1 +0xaf fp=0xc001123370 sp=0xc001123320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc0009d24e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0011233b0 sp=0xc001123370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001395590}, 0xc001321c68?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001123400 sp=0xc0011233b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001395590}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001123490 sp=0xc001123400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc00166bd10}, 0xc001321c68, 0xc0012b4d80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0011237d0 sp=0xc001123490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc00166bd10}, 0x3f252a0?, 0xc0015118d0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001123848 sp=0xc0011237d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000a89c20, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001123db0 sp=0xc001123848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000a89c20, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001123f28 sp=0xc001123db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001123fe0 sp=0xc001123f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001123fe8 sp=0xc001123fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 786 [select]: +runtime.gopark(0xc001a32fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001a32f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a32de8 sp=0xc001a32dc8 pc=0x4777ae +runtime.selectgo(0xc001a32fb0, 0xc001a32f70, 0xc000f40fd0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a32f38 sp=0xc001a32de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001a32fe0 sp=0xc001a32f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a32fe8 sp=0xc001a32fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 700 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 787 [select]: +runtime.gopark(0xc001b5bfb0?, 0x2?, 0xb0?, 0xbd?, 0xc001b5bee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b5bd58 sp=0xc001b5bd38 pc=0x4777ae +runtime.selectgo(0xc001b5bfb0, 0xc001b5bee0, 0x4aead9?, 0x0, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b5bea8 sp=0xc001b5bd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b5bfe0 sp=0xc001b5bea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b5bfe8 sp=0xc001b5bfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 700 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 788 [chan receive]: +runtime.gopark(0x20?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b34d60 sp=0xc001b34d40 pc=0x4777ae +runtime.chanrecv(0xc000c6b9e0, 0xc001b34e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b34dd8 sp=0xc001b34d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b34e00 sp=0xc001b34dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001b34fe0 sp=0xc001b34e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b34fe8 sp=0xc001b34fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 700 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 789 [select]: +runtime.gopark(0xc001fb1fa0?, 0x3?, 0xf0?, 0x1d?, 0xc001fb1f3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fb1da8 sp=0xc001fb1d88 pc=0x4777ae +runtime.selectgo(0xc001fb1fa0, 0xc001fb1f34, 0xc001982f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001fb1ef8 sp=0xc001fb1da8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:184 +0x1b3 fp=0xc001fb1fe0 sp=0xc001fb1ef8 pc=0x21557f3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fb1fe8 sp=0xc001fb1fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream in goroutine 700 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:182 +0x24a + +goroutine 775 [select]: +runtime.gopark(0xc002040a68?, 0x2?, 0x0?, 0x0?, 0xc002040a14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002040880 sp=0xc002040860 pc=0x4777ae +runtime.selectgo(0xc002040a68, 0xc002040a10, 0x7f429a44b300?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020409d0 sp=0xc002040880 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000b69cc0, {0xc001d8f000, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002040a98 sp=0xc0020409d0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000b69cc0, {0xc001d8f000, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002040b30 sp=0xc002040a98 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001d8e690, {0xc001d8f000, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002040b78 sp=0xc002040b30 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001d8e690}, {0xc001d8f000, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002040bc0 sp=0xc002040b78 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00165cc60, {0xc001d8f000, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002040c30 sp=0xc002040bc0 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001d8eff0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002040d00 sp=0xc002040c30 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc00165cc60, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002040df0 sp=0xc002040d00 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001d8eff0, {0x7f424347ae80, 0x779c980}, 0xc001ec2870?, {0x0, 0x0}, {0x43dd860, 0xc002338050}, 0x12b01b3?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002040ea8 sp=0xc002040df0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001ec2960, {0x43dd860?, 0xc002338050}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0020410e0 sp=0xc002040ea8 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001d8fe00, {0x43dd860, 0xc002338050}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0020411c8 sp=0xc0020410e0 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0007d3a20, {0x43dd860, 0xc002338050}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002041200 sp=0xc0020411c8 pc=0x1fd94b2 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamServer).Recv(0xc00143b120) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:167 +0x66 fp=0xc002041240 sp=0xc002041200 pc=0x2102886 +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream(0xc001267050, {0x5b512b8, 0xc00143b120}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:87 +0x218 fp=0xc002041370 sp=0xc002041240 pc=0x2154718 +github.com/hashicorp/go-plugin/internal/plugin._GRPCBroker_StartStream_Handler({0x41cb860?, 0xc001267050}, {0x5b4a6f0?, 0xc0007d3a20}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:148 +0xb6 fp=0xc0020413b0 sp=0xc002041370 pc=0x2102716 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41cb860, 0xc001267050}, {0x5b4ab30, 0xc001d8fe00}, 0xc001423e00?, 0x47efc98) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002041400 sp=0xc0020413b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41cb860, 0xc001267050}, {0x5b4ab30, 0xc001d8fe00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002041490 sp=0xc002041400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41cb860, 0xc001267050}, {0x5b492f8, 0xc001ec2960}, 0xc001423e00, 0xc001d3eb40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0020417d0 sp=0xc002041490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41cb860, 0xc001267050}, {0x5b492f8, 0xc001ec2960}, 0x3f252a0?, 0xc00143b050?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002041848 sp=0xc0020417d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165cc60, 0xc0012670e0, 0x770cb80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002041db0 sp=0xc002041848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165cc60, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002041f28 sp=0xc002041db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002041fe0 sp=0xc002041f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002041fe8 sp=0xc002041fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 782 [select]: +runtime.gopark(0xc00120a890?, 0x2?, 0x98?, 0xa7?, 0xc00120a83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00120a6a8 sp=0xc00120a688 pc=0x4777ae +runtime.selectgo(0xc00120a890, 0xc00120a838, 0x7f429a44b000?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00120a7f8 sp=0xc00120a6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001a0c5a0, {0xc001bba190, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc00120a8c0 sp=0xc00120a7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001a0c5a0, {0xc001bba190, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc00120a958 sp=0xc00120a8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001bba0c0, {0xc001bba190, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc00120a9a0 sp=0xc00120a958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001bba0c0}, {0xc001bba190, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00120a9e8 sp=0xc00120a9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00165d320, {0xc001bba190, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc00120aa58 sp=0xc00120a9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001bba180, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc00120ab28 sp=0xc00120aa58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001d926c0?, 0xc00165d320, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc00120ac18 sp=0xc00120ab28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001bba180, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001a0c8c0}, 0xc001266cc0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc00120acd0 sp=0xc00120ac18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001ec2a50, {0x43dcf60?, 0xc001a0c8c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc00120af08 sp=0xc00120acd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001bba2d0, {0x43dcf60, 0xc001a0c8c0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc00120aff0 sp=0xc00120af08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0007d3b40, {0x43dcf60, 0xc001a0c8c0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc00120b028 sp=0xc00120aff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00143b2c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc00120b068 sp=0xc00120b028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00143b2c0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc00120b0a8 sp=0xc00120b068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00143b2d0) + :1 +0x46 fp=0xc00120b0e0 sp=0xc00120b0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc00143b2d0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc00120b320 sp=0xc00120b0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc00143b2c0}) + :1 +0xaf fp=0xc00120b370 sp=0xc00120b320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc0007d3b40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc00120b3b0 sp=0xc00120b370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001bba2d0}, 0xc000580060?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc00120b400 sp=0xc00120b3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001bba2d0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc00120b490 sp=0xc00120b400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001ec2a50}, 0xc000580060, 0xc001d3f540) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc00120b7d0 sp=0xc00120b490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001ec2a50}, 0x3f252a0?, 0xc00143b1f0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc00120b848 sp=0xc00120b7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165d320, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc00120bdb0 sp=0xc00120b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165d320, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc00120bf28 sp=0xc00120bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00120bfe0 sp=0xc00120bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00120bfe8 sp=0xc00120bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 777 [select]: +runtime.gopark(0xc001b2ffb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b2ff74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b2fde8 sp=0xc001b2fdc8 pc=0x4777ae +runtime.selectgo(0xc001b2ffb0, 0xc001b2ff70, 0x3235202d20202020?, 0x0, 0x202d202020200a38?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b2ff38 sp=0xc001b2fde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b2ffe0 sp=0xc001b2ff38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b2ffe8 sp=0xc001b2ffe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 778 [select]: +runtime.gopark(0xc001b30fb0?, 0x2?, 0xb0?, 0xd?, 0xc001b30ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b30d58 sp=0xc001b30d38 pc=0x4777ae +runtime.selectgo(0xc001b30fb0, 0xc001b30ee0, 0x5d53d0?, 0x0, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b30ea8 sp=0xc001b30d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b30fe0 sp=0xc001b30ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b30fe8 sp=0xc001b30fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 779 [chan receive]: +runtime.gopark(0xb626f746f72702f?, 0x2e7974706d652f66?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a37d60 sp=0xc001a37d40 pc=0x4777ae +runtime.chanrecv(0xc00119ac00, 0xc001a37e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001a37dd8 sp=0xc001a37d60 pc=0x441325 +runtime.chanrecv1(0x28?, 0x14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001a37e00 sp=0xc001a37dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001a37fe0 sp=0xc001a37e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a37fe8 sp=0xc001a37fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 780 [select]: +runtime.gopark(0xc001a38fa0?, 0x3?, 0x0?, 0x4e?, 0xc001a38f3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a38da8 sp=0xc001a38d88 pc=0x4777ae +runtime.selectgo(0xc001a38fa0, 0xc001a38f34, 0xc001a38f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a38ef8 sp=0xc001a38da8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:73 +0x1ad fp=0xc001a38fe0 sp=0xc001a38ef8 pc=0x2154aad +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a38fe8 sp=0xc001a38fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream in goroutine 775 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:71 +0x1d1 + +goroutine 760 [select]: +runtime.gopark(0xc001976890?, 0x2?, 0x98?, 0x67?, 0xc00197683c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019766a8 sp=0xc001976688 pc=0x4777ae +runtime.selectgo(0xc001976890, 0xc001976838, 0x7f429a451f00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0019767f8 sp=0xc0019766a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0014353b0, {0xc001e53e70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0019768c0 sp=0xc0019767f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0014353b0, {0xc001e53e70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001976958 sp=0xc0019768c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001e53d40, {0xc001e53e70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0019769a0 sp=0xc001976958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001e53d40}, {0xc001e53e70, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0019769e8 sp=0xc0019769a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008a2240, {0xc001e53e70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001976a58 sp=0xc0019769e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001e53e60, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001976b28 sp=0xc001976a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0019f1340?, 0xc0008a2240, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001976c18 sp=0xc001976b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001e53e60, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001435a40}, 0xc001266cc0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001976cd0 sp=0xc001976c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001344a50, {0x43dcf60?, 0xc001435a40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001976f08 sp=0xc001976cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0020861e0, {0x43dcf60, 0xc001435a40}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001976ff0 sp=0xc001976f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001f0fd80, {0x43dcf60, 0xc001435a40}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001977028 sp=0xc001976ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00114d7e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001977068 sp=0xc001977028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00114d7e0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0019770a8 sp=0xc001977068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00114d7f0) + :1 +0x46 fp=0xc0019770e0 sp=0xc0019770a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc00114d7f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001977320 sp=0xc0019770e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc00114d7e0}) + :1 +0xaf fp=0xc001977370 sp=0xc001977320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc001f0fd80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0019773b0 sp=0xc001977370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc0020861e0}, 0xc0004a7260?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001977400 sp=0xc0019773b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc0020861e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001977490 sp=0xc001977400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001344a50}, 0xc0004a7260, 0xc0019f0740) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0019777d0 sp=0xc001977490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001344a50}, 0x3f252a0?, 0xc00114d6f0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001977848 sp=0xc0019777d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0008a2240, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001977db0 sp=0xc001977848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0008a2240, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001977f28 sp=0xc001977db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001977fe0 sp=0xc001977f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001977fe8 sp=0xc001977fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 784 [select]: +runtime.gopark(0xc001b31fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b31f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b31de8 sp=0xc001b31dc8 pc=0x4777ae +runtime.selectgo(0xc001b31fb0, 0xc001b31f70, 0xc0017fac00?, 0x0, 0xc00119af60?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b31f38 sp=0xc001b31de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b31fe0 sp=0xc001b31f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b31fe8 sp=0xc001b31fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 785 [select]: +runtime.gopark(0xc001b3bfb0?, 0x2?, 0xb0?, 0xbd?, 0xc001b3bee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b3bd58 sp=0xc001b3bd38 pc=0x4777ae +runtime.selectgo(0xc001b3bfb0, 0xc001b3bee0, 0x5d53d0?, 0x0, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b3bea8 sp=0xc001b3bd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b3bfe0 sp=0xc001b3bea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b3bfe8 sp=0xc001b3bfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 802 [chan receive]: +runtime.gopark(0x3332343637313138?, 0x7d2265686361632f?, 0x2c?, 0x22?, 0x223a22656e69676e?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b2cd60 sp=0xc001b2cd40 pc=0x4777ae +runtime.chanrecv(0xc00119b5c0, 0xc001b2ce60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b2cdd8 sp=0xc001b2cd60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b2ce00 sp=0xc001b2cdd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001b2cfe0 sp=0xc001b2ce00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b2cfe8 sp=0xc001b2cfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 807 [select]: +runtime.gopark(0xc001b2dfb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b2df74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b2dde8 sp=0xc001b2ddc8 pc=0x4777ae +runtime.selectgo(0xc001b2dfb0, 0xc001b2df70, 0x0?, 0x0, 0x202d202020000000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b2df38 sp=0xc001b2dde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b2dfe0 sp=0xc001b2df38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b2dfe8 sp=0xc001b2dfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 804 [select]: +runtime.gopark(0xc001b4c890?, 0x2?, 0x98?, 0xc7?, 0xc001b4c83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b4c6a8 sp=0xc001b4c688 pc=0x4777ae +runtime.selectgo(0xc001b4c890, 0xc001b4c838, 0x7f429a451000?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b4c7f8 sp=0xc001b4c6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001a0d040, {0xc001bbb210, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001b4c8c0 sp=0xc001b4c7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001a0d040, {0xc001bbb210, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001b4c958 sp=0xc001b4c8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001bbadb0, {0xc001bbb210, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001b4c9a0 sp=0xc001b4c958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001bbadb0}, {0xc001bbb210, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001b4c9e8 sp=0xc001b4c9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00165dc20, {0xc001bbb210, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001b4ca58 sp=0xc001b4c9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001bbb200, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001b4cb28 sp=0xc001b4ca58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001a28bc0?, 0xc00165dc20, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001b4cc18 sp=0xc001b4cb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001bbb200, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001a0d630}, 0xc001266cc0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001b4ccd0 sp=0xc001b4cc18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001ec2b40, {0x43dcf60?, 0xc001a0d630}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001b4cf08 sp=0xc001b4ccd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001bbb350, {0x43dcf60, 0xc001a0d630}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001b4cff0 sp=0xc001b4cf08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc00127c1e0, {0x43dcf60, 0xc001a0d630}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001b4d028 sp=0xc001b4cff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00143b8b0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001b4d068 sp=0xc001b4d028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00143b8b0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001b4d0a8 sp=0xc001b4d068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00143b8c0) + :1 +0x46 fp=0xc001b4d0e0 sp=0xc001b4d0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc00143b8c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001b4d320 sp=0xc001b4d0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc00143b8b0}) + :1 +0xaf fp=0xc001b4d370 sp=0xc001b4d320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc00127c1e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001b4d3b0 sp=0xc001b4d370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001bbb350}, 0xc000581020?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001b4d400 sp=0xc001b4d3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001bbb350}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001b4d490 sp=0xc001b4d400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001ec2b40}, 0xc000581020, 0xc001d93fc0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001b4d7d0 sp=0xc001b4d490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001ec2b40}, 0x3f252a0?, 0xc00143b7e0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001b4d848 sp=0xc001b4d7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165dc20, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001b4ddb0 sp=0xc001b4d848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00165dc20, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001b4df28 sp=0xc001b4ddb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001b4dfe0 sp=0xc001b4df28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b4dfe8 sp=0xc001b4dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 808 [select]: +runtime.gopark(0xc001b2efb0?, 0x2?, 0xb0?, 0xed?, 0xc001b2eee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b2ed58 sp=0xc001b2ed38 pc=0x4777ae +runtime.selectgo(0xc001b2efb0, 0xc001b2eee0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b2eea8 sp=0xc001b2ed58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b2efe0 sp=0xc001b2eea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b2efe8 sp=0xc001b2efe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 809 [chan receive]: +runtime.gopark(0x0?, 0x3?, 0x11?, 0x0?, 0x4db?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b37d60 sp=0xc001b37d40 pc=0x4777ae +runtime.chanrecv(0xc00119bf20, 0xc001b37e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b37dd8 sp=0xc001b37d60 pc=0x441325 +runtime.chanrecv1(0xc?, 0x4dc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b37e00 sp=0xc001b37dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001b37fe0 sp=0xc001b37e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b37fe8 sp=0xc001b37fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1134 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc001ee32b0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00274cdd8 sp=0xc00274cdb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc000ef5450, 0x2) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc00274ce28 sp=0xc00274cdd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc000ef5440) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc00274ce58 sp=0xc00274ce28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc0020ace00, 0xc001ee32b0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc00274cfb0 sp=0xc00274ce58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc00274cfe0 sp=0xc00274cfb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00274cfe8 sp=0xc00274cfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 1133 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 811 [select]: +runtime.gopark(0xc001b38fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b38f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b38de8 sp=0xc001b38dc8 pc=0x4777ae +runtime.selectgo(0xc001b38fb0, 0xc001b38f70, 0xc0017fbaa0?, 0x0, 0xc00119bf80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b38f38 sp=0xc001b38de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b38fe0 sp=0xc001b38f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b38fe8 sp=0xc001b38fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 810 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 812 [select]: +runtime.gopark(0xc001b39fb0?, 0x2?, 0x0?, 0x0?, 0xc001b39ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b39d58 sp=0xc001b39d38 pc=0x4777ae +runtime.selectgo(0xc001b39fb0, 0xc001b39ee0, 0x5d53d0?, 0x0, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b39ea8 sp=0xc001b39d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b39fe0 sp=0xc001b39ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b39fe8 sp=0xc001b39fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 810 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 813 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b3ad60 sp=0xc001b3ad40 pc=0x4777ae +runtime.chanrecv(0xc0014365a0, 0xc001b3ae60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b3add8 sp=0xc001b3ad60 pc=0x441325 +runtime.chanrecv1(0x8?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b3ae00 sp=0xc001b3add8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001b3afe0 sp=0xc001b3ae00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b3afe8 sp=0xc001b3afe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 810 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 790 [select]: +runtime.gopark(0xc001f2a890?, 0x2?, 0x4e?, 0x1a?, 0xc001f2a83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f2a6a8 sp=0xc001f2a688 pc=0x4777ae +runtime.selectgo(0xc001f2a890, 0xc001f2a838, 0x7f429a44fb00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f2a7f8 sp=0xc001f2a6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00128c8c0, {0xc0013958a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001f2a8c0 sp=0xc001f2a7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00128c8c0, {0xc0013958a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001f2a958 sp=0xc001f2a8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001924930, {0xc0013958a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001f2a9a0 sp=0xc001f2a958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001924930}, {0xc0013958a0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001f2a9e8 sp=0xc001f2a9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0015f9b00, {0xc0013958a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001f2aa58 sp=0xc001f2a9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001395890, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001f2ab28 sp=0xc001f2aa58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001ab87c0?, 0xc0015f9b00, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001f2ac18 sp=0xc001f2ab28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001395890, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0007c18b0}, 0xc001266cc0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001f2acd0 sp=0xc001f2ac18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc00166be00, {0x43dcf60?, 0xc0007c18b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001f2af08 sp=0xc001f2acd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001395a10, {0x43dcf60, 0xc0007c18b0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001f2aff0 sp=0xc001f2af08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0009d2560, {0x43dcf60, 0xc0007c18b0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001f2b028 sp=0xc001f2aff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc001511ae0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001f2b068 sp=0xc001f2b028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc001511ae0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001f2b0a8 sp=0xc001f2b068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc001511af0) + :1 +0x46 fp=0xc001f2b0e0 sp=0xc001f2b0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc001511af0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001f2b320 sp=0xc001f2b0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc001511ae0}) + :1 +0xaf fp=0xc001f2b370 sp=0xc001f2b320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc0009d2560}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001f2b3b0 sp=0xc001f2b370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001395a10}, 0xc001321e00?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001f2b400 sp=0xc001f2b3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001395a10}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001f2b490 sp=0xc001f2b400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc00166be00}, 0xc001321e00, 0xc0012b5a80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001f2b7d0 sp=0xc001f2b490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc00166be00}, 0x3f252a0?, 0xc001511a10?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001f2b848 sp=0xc001f2b7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0015f9b00, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001f2bdb0 sp=0xc001f2b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0015f9b00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001f2bf28 sp=0xc001f2bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001f2bfe0 sp=0xc001f2bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f2bfe8 sp=0xc001f2bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 757 [select]: +runtime.gopark(0xc001a33fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001a33f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a33de8 sp=0xc001a33dc8 pc=0x4777ae +runtime.selectgo(0xc001a33fb0, 0xc001a33f70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a33f38 sp=0xc001a33de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001a33fe0 sp=0xc001a33f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a33fe8 sp=0xc001a33fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 765 [chan receive]: +runtime.gopark(0xc000b3d020?, 0x27?, 0x0?, 0x0?, 0x460fed5?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b57ef8 sp=0xc001b57ed8 pc=0x4777ae +runtime.chanrecv(0xc001143200, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b57f70 sp=0xc001b57ef8 pc=0x441325 +runtime.chanrecv1(0xc001fefae0?, 0x47e975?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b57f98 sp=0xc001b57f70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001b57fe0 sp=0xc001b57f98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b57fe8 sp=0xc001b57fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 763 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 820 [select]: +runtime.gopark(0xc001bca890?, 0x2?, 0x98?, 0xa7?, 0xc001bca83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bca6a8 sp=0xc001bca688 pc=0x4777ae +runtime.selectgo(0xc001bca890, 0xc001bca838, 0x7f429a454c00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bca7f8 sp=0xc001bca6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0007c1e50, {0xc001dcf360, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001bca8c0 sp=0xc001bca7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0007c1e50, {0xc001dcf360, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001bca958 sp=0xc001bca8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001dcebd0, {0xc001dcf360, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001bca9a0 sp=0xc001bca958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001dcebd0}, {0xc001dcf360, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001bca9e8 sp=0xc001bca9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000e50900, {0xc001dcf360, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001bcaa58 sp=0xc001bca9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001dcf350, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001bcab28 sp=0xc001bcaa58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001ab9680?, 0xc000e50900, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001bcac18 sp=0xc001bcab28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001dcf350, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001aba4b0}, 0xc0010e4480?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001bcacd0 sp=0xc001bcac18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc00166bef0, {0x43dcf60?, 0xc001aba4b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001bcaf08 sp=0xc001bcacd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001dcf4a0, {0x43dcf60, 0xc001aba4b0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001bcaff0 sp=0xc001bcaf08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0009d2f40, {0x43dcf60, 0xc001aba4b0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001bcb028 sp=0xc001bcaff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc001511f20) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001bcb068 sp=0xc001bcb028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc001511f20?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001bcb0a8 sp=0xc001bcb068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc001511f30) + :1 +0x46 fp=0xc001bcb0e0 sp=0xc001bcb0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc001511f30}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001bcb320 sp=0xc001bcb0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc001511f20}) + :1 +0xaf fp=0xc001bcb370 sp=0xc001bcb320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc0009d2f40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001bcb3b0 sp=0xc001bcb370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001dcf4a0}, 0xc0012e4390?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001bcb400 sp=0xc001bcb3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001dcf4a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001bcb490 sp=0xc001bcb400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc00166bef0}, 0xc0012e4390, 0xc001ab8c80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001bcb7d0 sp=0xc001bcb490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc00166bef0}, 0x3f252a0?, 0xc001511e50?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001bcb848 sp=0xc001bcb7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000e50900, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001bcbdb0 sp=0xc001bcb848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000e50900, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001bcbf28 sp=0xc001bcbdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001bcbfe0 sp=0xc001bcbf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bcbfe8 sp=0xc001bcbfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 801 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x20?, 0x3?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001997cd0 sp=0xc001997cb0 pc=0x4777ae +runtime.chanrecv(0xc00197e8a0, 0xc001997de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001997d48 sp=0xc001997cd0 pc=0x441325 +runtime.chanrecv1(0xc001997df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001997d70 sp=0xc001997d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001997f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001997e00 sp=0xc001997d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc001fefdb0, 0x420c60?, 0xc001924e70) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001997fa0 sp=0xc001997e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001997fe0 sp=0xc001997fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001997fe8 sp=0xc001997fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 798 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 763 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0xc3?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b13cd0 sp=0xc001b13cb0 pc=0x4777ae +runtime.chanrecv(0xc001c187e0, 0xc001b13de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b13d48 sp=0xc001b13cd0 pc=0x441325 +runtime.chanrecv1(0xc001b13df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b13d70 sp=0xc001b13d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001b13f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001b13e00 sp=0xc001b13d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc001fefdb0, 0x0?, 0xc001c163c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001b13fa0 sp=0xc001b13e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001b13fe0 sp=0xc001b13fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b13fe8 sp=0xc001b13fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 762 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 1378 [select]: +runtime.gopark(0xc0024d1f70?, 0x2?, 0xb4?, 0x1b?, 0xc0024d1ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024d1d20 sp=0xc0024d1d00 pc=0x4777ae +runtime.selectgo(0xc0024d1f70, 0xc0024d1eb8, 0x1fa3b79?, 0x0, 0x5b50208?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0024d1e70 sp=0xc0024d1d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d86a80, {0x5b426e0, 0xc002338c80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0024d1fa0 sp=0xc0024d1e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0024d1fe0 sp=0xc0024d1fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024d1fe8 sp=0xc0024d1fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1361 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1132 [select]: +runtime.gopark(0xc002057fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002057f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002057de8 sp=0xc002057dc8 pc=0x4777ae +runtime.selectgo(0xc002057fb0, 0xc002057f70, 0x4aead9?, 0x0, 0x23220cf?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002057f38 sp=0xc002057de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002057fe0 sp=0xc002057f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002057fe8 sp=0xc002057fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1138 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 794 [select]: +runtime.gopark(0xc001b6dfb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b6df74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b6dde8 sp=0xc001b6ddc8 pc=0x4777ae +runtime.selectgo(0xc001b6dfb0, 0xc001b6df70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b6df38 sp=0xc001b6dde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b6dfe0 sp=0xc001b6df38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b6dfe8 sp=0xc001b6dfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 795 [select]: +runtime.gopark(0xc001b54fb0?, 0x2?, 0xb0?, 0x4d?, 0xc001b54ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b54d58 sp=0xc001b54d38 pc=0x4777ae +runtime.selectgo(0xc001b54fb0, 0xc001b54ee0, 0xe?, 0x0, 0x8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b54ea8 sp=0xc001b54d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b54fe0 sp=0xc001b54ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b54fe8 sp=0xc001b54fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 796 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b55d60 sp=0xc001b55d40 pc=0x4777ae +runtime.chanrecv(0xc001088060, 0xc001b55e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b55dd8 sp=0xc001b55d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b55e00 sp=0xc001b55dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001b55fe0 sp=0xc001b55e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b55fe8 sp=0xc001b55fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 834 [select]: +runtime.gopark(0xc001b7b448?, 0x2?, 0xd0?, 0xb2?, 0xc001b7b3fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b7b278 sp=0xc001b7b258 pc=0x4777ae +runtime.selectgo(0xc001b7b448, 0xc001b7b3f8, 0xc0015113b8?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b7b3c8 sp=0xc001b7b278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00094f0e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001b7b478 sp=0xc001b7b3c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00194c750, {0x42219c0?, 0xc001924f00}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc001b7b6d8 sp=0xc001b7b478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00094e414?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001b7b738 sp=0xc001b7b6d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00094e360, 0xc001b7b8f0, 0xc001b7b8c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc001b7b7e0 sp=0xc001b7b738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00094e360, {0x42219c0, 0xc001924f00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001b7b920 sp=0xc001b7b7e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc001924fc0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc001924ed0}, {0x42219c0, 0xc001924f00}, 0x0?, {0xc001f0f380, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc001b7b9c0 sp=0xc001b7b920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc001fefae0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc001924ed0}, {0x42219c0, 0xc001924f00}, 0xc001838400, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc001b7bd30 sp=0xc001b7b9c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc001838400, {0x5b426e0, 0xc001fefae0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc001924ed0}, {0x42219c0, 0xc001924f00}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc001b7be58 sp=0xc001b7bd30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseManagementAPI(0xc00158bb70, {0x5b426e0, 0xc001fefae0}, 0x0?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:51 +0xee fp=0xc001b7bee8 sp=0xc001b7be58 pc=0x21ab66e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI.func2(0x1) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:190 +0xf2 fp=0xc001b7bf58 sp=0xc001b7bee8 pc=0x21a7f32 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc001b7bfe0 sp=0xc001b7bf58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b7bfe8 sp=0xc001b7bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 798 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 798 [select]: +runtime.gopark(0xc001b56f28?, 0x2?, 0xb4?, 0x1b?, 0xc001b56ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b56d30 sp=0xc001b56d10 pc=0x4777ae +runtime.selectgo(0xc001b56f28, 0xc001b56eb8, 0x0?, 0x0, 0xc001b56f20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b56e80 sp=0xc001b56d30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc001924a80, 0xc000f1ff08, 0xc00158bd10) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001b56f58 sp=0xc001b56e80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI(0xc001924a80, {0x5b6f770?, 0xc0010c0e70}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:185 +0x10a fp=0xc001b56f98 sp=0xc001b56f58 pc=0x21a7e0a +github.com/rancher/opni/pkg/management.NewServer.func1.2() + /home/ec2-user/opni/pkg/management/server.go:156 +0x62 fp=0xc001b56fe0 sp=0xc001b56f98 pc=0x231c0c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b56fe8 sp=0xc001b56fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 797 + /home/ec2-user/opni/pkg/management/server.go:156 +0x137 + +goroutine 799 [select]: +runtime.gopark(0xc001b58f28?, 0x2?, 0xb4?, 0x1b?, 0xc001b58ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b58d30 sp=0xc001b58d10 pc=0x4777ae +runtime.selectgo(0xc001b58f28, 0xc001b58eb8, 0x22c21f5?, 0x0, 0xc001b58f20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b58e80 sp=0xc001b58d30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc001924a80, 0xc000581950, 0xc00143bd30) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001b58f58 sp=0xc001b58e80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer(0xc001924a80, {0x5b21b70?, 0xc0017e1740}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:198 +0x10a fp=0xc001b58f98 sp=0xc001b58f58 pc=0x21a810a +github.com/rancher/opni/pkg/management.NewServer.func1.3() + /home/ec2-user/opni/pkg/management/server.go:158 +0x62 fp=0xc001b58fe0 sp=0xc001b58f98 pc=0x231c022 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b58fe8 sp=0xc001b58fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 797 + /home/ec2-user/opni/pkg/management/server.go:158 +0x245 + +goroutine 1309 [IO wait]: +runtime.gopark(0xc0024d6fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024d6f98 sp=0xc0024d6f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0024d6fd0 sp=0xc0024d6f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d580, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0024d6ff0 sp=0xc0024d6fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0025f2020, 0xc002d73c00?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0024d7038 sp=0xc0024d6ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0025f2000, {0xc002d73c00, 0xc00, 0xc00}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0024d7128 sp=0xc0024d7038 pc=0x50e5a5 +net.(*netFD).Read(0xc0025f2000, {0xc002d73c00, 0xc00, 0xc00}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0024d7188 sp=0xc0024d7128 pc=0x89342b +net.(*conn).Read(0xc001d6efb8, {0xc002d73c00, 0xc00, 0xc00}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0024d7210 sp=0xc0024d7188 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6efb8, {0xc002d73c00, 0xc00, 0xc00}) + :1 +0x54 fp=0xc0024d7258 sp=0xc0024d7210 pc=0x8d72f4 +crypto/tls.(*atLeastReader).Read(0xc00304a2b8, {0xc002d73c00, 0xc00, 0xc00}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0024d72b8 sp=0xc0024d7258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00133a2a8, {0x5b0f6c0, 0xc00304a2b8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0024d7320 sp=0xc0024d72b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00133a000, {0x5b0f820?, 0xc001d6efb8}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0024d7390 sp=0xc0024d7320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00133a000, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0024d79c0 sp=0xc0024d7390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00133a000, {0xc00265c000, 0x8000, 0xc0024d7af8?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0024d7aa8 sp=0xc0024d79c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0015ff7c0, {0xc00265c000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0024d7af0 sp=0xc0024d7aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc002081980, {0xc000b9a740, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0024d7bb0 sp=0xc0024d7af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002081980}, {0xc000b9a740, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0024d7bf8 sp=0xc0024d7bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000b9a740, 0x9, 0x9}, {0x5b0f0a0, 0xc002081980}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0024d7c98 sp=0xc0024d7bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000b9a700) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0024d7dd0 sp=0xc0024d7c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000adcfc0, 0xc0018534a0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0024d7fb0 sp=0xc0024d7dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0024d7fe0 sp=0xc0024d7fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024d7fe8 sp=0xc0024d7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1353 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 758 [select]: +runtime.gopark(0xc001b69fb0?, 0x2?, 0xb0?, 0x9d?, 0xc001b69ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b69d58 sp=0xc001b69d38 pc=0x4777ae +runtime.selectgo(0xc001b69fb0, 0xc001b69ee0, 0x69747265706f7270?, 0x0, 0x448845?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b69ea8 sp=0xc001b69d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b69fe0 sp=0xc001b69ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b69fe8 sp=0xc001b69fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 759 [chan receive]: +runtime.gopark(0x632f0a1a2a013a0f?, 0x6572756769666e6f?, 0x12?, 0x72?, 0x6e6f437472656c41?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a34d60 sp=0xc001a34d40 pc=0x4777ae +runtime.chanrecv(0xc001678780, 0xc001a34e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001a34dd8 sp=0xc001a34d60 pc=0x441325 +runtime.chanrecv1(0x2e676e697472656c?, 0x6f697469646e6f43?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001a34e00 sp=0xc001a34dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001a34fe0 sp=0xc001a34e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a34fe8 sp=0xc001a34fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 815 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xe0?, 0x6?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b0fcd0 sp=0xc001b0fcb0 pc=0x4777ae +runtime.chanrecv(0xc00197e9c0, 0xc001b0fde0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b0fd48 sp=0xc001b0fcd0 pc=0x441325 +runtime.chanrecv1(0xc001b0fdf0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b0fd70 sp=0xc001b0fd48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001b0ff30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001b0fe00 sp=0xc001b0fd70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc001fefdb0, 0x2?, 0xc001b525d0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001b0ffa0 sp=0xc001b0fe00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001b0ffe0 sp=0xc001b0ffa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b0ffe8 sp=0xc001b0ffe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 799 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 816 [select]: +runtime.gopark(0xc001b77448?, 0x2?, 0xd0?, 0x72?, 0xc001b773fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b77278 sp=0xc001b77258 pc=0x4777ae +runtime.selectgo(0xc001b77448, 0xc001b773f8, 0xc0015113b8?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b773c8 sp=0xc001b77278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00110efc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001b77478 sp=0xc001b773c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc0017fdad0, {0x42219c0?, 0xc001b52a50}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc001b776d8 sp=0xc001b77478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00110ead4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001b77738 sp=0xc001b776d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00110ea20, 0xc001b778f0, 0xc001b778c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc001b777e0 sp=0xc001b77738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00110ea20, {0x42219c0, 0xc001b52a50}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001b77920 sp=0xc001b777e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc001b52b10}, {0x4673e80, 0x23}, {0x4278de0, 0xc001b52a20}, {0x42219c0, 0xc001b52a50}, 0x0?, {0xc001f0f380, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc001b779c0 sp=0xc001b77920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc001fefae0}, {0x4673e80, 0x23}, {0x4278de0, 0xc001b52a20}, {0x42219c0, 0xc001b52a50}, 0xc001838400, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc001b77d30 sp=0xc001b779c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc001838400, {0x5b426e0, 0xc001fefae0}, {0x4673e80, 0x23}, {0x4278de0, 0xc001b52a20}, {0x42219c0, 0xc001b52a50}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc001b77e58 sp=0xc001b77d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseNodeManagerClient(0xc00158bb70, {0x5b426e0, 0xc001fefae0}, 0xc001707750?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:60 +0xee fp=0xc001b77ee8 sp=0xc001b77e58 pc=0x21ab7ee +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer.func2(0x2) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:203 +0xf2 fp=0xc001b77f58 sp=0xc001b77ee8 pc=0x21a8232 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc001b77fe0 sp=0xc001b77f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b77fe8 sp=0xc001b77fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 799 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 835 [chan receive]: +runtime.gopark(0x22c2125?, 0xc000081750?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b35ef8 sp=0xc001b35ed8 pc=0x4777ae +runtime.chanrecv(0xc001143200, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b35f70 sp=0xc001b35ef8 pc=0x441325 +runtime.chanrecv1(0xc001fefae0?, 0x5b6f770?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b35f98 sp=0xc001b35f70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001b35fe0 sp=0xc001b35f98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b35fe8 sp=0xc001b35fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 801 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 836 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bd7978 sp=0xc001bd7958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001bd79b0 sp=0xc001bd7978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163c200, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001bd79d0 sp=0xc001bd79b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193ab20, 0xc001bd7a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001bd7a18 sp=0xc001bd79d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00193ab00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001bd7b08 sp=0xc001bd7a18 pc=0x513f65 +net.(*netFD).accept(0xc00193ab00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001bd7c00 sp=0xc001bd7b08 pc=0x8967be +net.(*UnixListener).accept(0xc001924ea0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001bd7c40 sp=0xc001bd7c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc001924ea0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001bd7cb0 sp=0xc001bd7c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc001dedf00) + :1 +0x44 fp=0xc001bd7ce8 sp=0xc001bd7cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc00191c780, {0x5b2c6a0, 0xc001dedf00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001bd7f18 sp=0xc001bd7ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc001bd7f68 sp=0xc001bd7f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc000b30200?, 0xc00158bf40?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001bd7fa8 sp=0xc001bd7f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001bd7fe0 sp=0xc001bd7fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bd7fe8 sp=0xc001bd7fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 801 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 837 [select]: +runtime.gopark(0xc001962f38?, 0x2?, 0x69?, 0x97?, 0xc001962f14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001962d90 sp=0xc001962d70 pc=0x4777ae +runtime.selectgo(0xc001962f38, 0xc001962f10, 0xc001962f18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001962ee0 sp=0xc001962d90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc001962f68 sp=0xc001962ee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc000126198?, 0xc0018be000?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001962fa8 sp=0xc001962f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001962fe0 sp=0xc001962fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001962fe8 sp=0xc001962fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 801 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 838 [chan receive]: +runtime.gopark(0x0?, 0x2?, 0x1?, 0xd6?, 0xc000086768?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d4bef8 sp=0xc001d4bed8 pc=0x4777ae +runtime.chanrecv(0xc001143200, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001d4bf70 sp=0xc001d4bef8 pc=0x441325 +runtime.chanrecv1(0xc001fefae0?, 0x2106c90?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001d4bf98 sp=0xc001d4bf70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001d4bfe0 sp=0xc001d4bf98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d4bfe8 sp=0xc001d4bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 815 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 839 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bd9978 sp=0xc001bd9958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001bd99b0 sp=0xc001bd9978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163c108, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001bd99d0 sp=0xc001bd99b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0017cf1a0, 0xc001bd9a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001bd9a18 sp=0xc001bd99d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0017cf180) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001bd9b08 sp=0xc001bd9a18 pc=0x513f65 +net.(*netFD).accept(0xc0017cf180) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001bd9c00 sp=0xc001bd9b08 pc=0x8967be +net.(*UnixListener).accept(0xc001b529f0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001bd9c40 sp=0xc001bd9c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc001b529f0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001bd9cb0 sp=0xc001bd9c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc00127c6a0) + :1 +0x44 fp=0xc001bd9ce8 sp=0xc001bd9cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc00191c960, {0x5b2c6a0, 0xc00127c6a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001bd9f18 sp=0xc001bd9ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc001bd9f68 sp=0xc001bd9f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc000b30620?, 0xc0018be140?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001bd9fa8 sp=0xc001bd9f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001bd9fe0 sp=0xc001bd9fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bd9fe8 sp=0xc001bd9fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 815 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 840 [select]: +runtime.gopark(0xc001b36f38?, 0x2?, 0x2?, 0x0?, 0xc001b36f14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b36d90 sp=0xc001b36d70 pc=0x4777ae +runtime.selectgo(0xc001b36f38, 0xc001b36f10, 0xc001b36f18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b36ee0 sp=0xc001b36d90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc001b36f68 sp=0xc001b36ee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc000126258?, 0xc0018be160?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001b36fa8 sp=0xc001b36f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001b36fe0 sp=0xc001b36fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b36fe8 sp=0xc001b36fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 815 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 823 [chan receive]: +runtime.gopark(0xc001ded584?, 0x4be6c0?, 0x80?, 0xd5?, 0x4be720?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00272ecf8 sp=0xc00272ecd8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00272ed70 sp=0xc00272ecf8 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0xc00290dbf0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00272ed98 sp=0xc00272ed70 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/gateway.(*Plugin).UseManagementAPI(0xc000379180, {0x5b6f168, 0xc001854f90}) + /home/ec2-user/opni/plugins/metrics/pkg/gateway/system.go:69 +0x67c fp=0xc00272efa0 sp=0xc00272ed98 pc=0x3db3f9c +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseManagementAPI(0xc001160340, {0x4b05a9?, 0xc001b0d0f0?}, 0xc002087aa0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:102 +0x30f fp=0xc00272f098 sp=0xc00272efa0 pc=0x21a640f +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler.func1({0x5b426a8, 0xc001c16000}, {0x4278de0?, 0xc002087aa0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:151 +0x83 fp=0xc00272f0f0 sp=0xc00272f098 pc=0x21ac063 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc001c16000}, {0x4278de0, 0xc002087aa0}, 0xc001f0fe00, 0xc0004a78c0) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc00272f148 sp=0xc00272f0f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc001c16000}, {0x4278de0, 0xc002087aa0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc00272f1e8 sp=0xc00272f148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc002087680}, {0x4278de0, 0xc002087aa0}, 0xc001f0fe00, 0xc0019f1480) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc00272f6b8 sp=0xc00272f1e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc002087680}, {0x4278de0, 0xc002087aa0}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc00272f740 sp=0xc00272f6b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler({0x4278ea0?, 0xc001160340}, {0x5b426a8, 0xc002087680}, 0xc002012f50, 0xc001ded8a0) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:153 +0x1c4 fp=0xc00272f7d0 sp=0xc00272f740 pc=0x21abec4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000e510e0, 0xc0010e57d0, 0x771eca0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc00272fdb0 sp=0xc00272f7d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000e510e0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc00272ff28 sp=0xc00272fdb0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00272ffe0 sp=0xc00272ff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00272ffe8 sp=0xc00272ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 824 [chan receive]: +runtime.gopark(0x4be6c0?, 0xc0014482d0?, 0x20?, 0xe7?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e36f08 sp=0xc001e36ee8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001e36f80 sp=0xc001e36f08 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x5b11a20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001e36fa8 sp=0xc001e36f80 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/gateway.(*Plugin).UseNodeManagerClient(0xc000379180, {0x5b11a20, 0xc0018552b0}) + /home/ec2-user/opni/plugins/metrics/pkg/gateway/system.go:74 +0x93 fp=0xc001e36fd0 sp=0xc001e36fa8 pc=0x3db4593 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseNodeManagerClient(0xc001160340, {0x4b05a9?, 0xc001a0b0f0?}, 0xc001ac23c0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:113 +0x18d fp=0xc001e37098 sp=0xc001e36fd0 pc=0x21a68cd +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler.func1({0x5b426a8, 0xc001ac2480}, {0x4278de0?, 0xc001ac23c0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:169 +0x83 fp=0xc001e370f0 sp=0xc001e37098 pc=0x21ac443 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc001ac2480}, {0x4278de0, 0xc001ac23c0}, 0xc0009d3240, 0xc0012e46d8) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001e37148 sp=0xc001e370f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc001ac2480}, {0x4278de0, 0xc001ac23c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc001e371e8 sp=0xc001e37148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc001ac2390}, {0x4278de0, 0xc001ac23c0}, 0xc0009d3240, 0xc001ab9840) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc001e376b8 sp=0xc001e371e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc001ac2390}, {0x4278de0, 0xc001ac23c0}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001e37740 sp=0xc001e376b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler({0x4278ea0?, 0xc001160340}, {0x5b426a8, 0xc001ac2390}, 0xc0018b8380, 0xc001ded8a0) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:171 +0x1c4 fp=0xc001e377d0 sp=0xc001e37740 pc=0x21ac2a4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000e51200, 0xc0010e57d0, 0x771ecb8, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001e37db0 sp=0xc001e377d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc000e51200, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001e37f28 sp=0xc001e37db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001e37fe0 sp=0xc001e37f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e37fe8 sp=0xc001e37fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 850 [select]: +runtime.gopark(0xc001b32fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b32f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b32de8 sp=0xc001b32dc8 pc=0x4777ae +runtime.selectgo(0xc001b32fb0, 0xc001b32f70, 0xc0017077d0?, 0x0, 0xc000000002?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b32f38 sp=0xc001b32de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b32fe0 sp=0xc001b32f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b32fe8 sp=0xc001b32fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 851 [select]: +runtime.gopark(0xc001b5afb0?, 0x2?, 0xb0?, 0xad?, 0xc001b5aee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b5ad58 sp=0xc001b5ad38 pc=0x4777ae +runtime.selectgo(0xc001b5afb0, 0xc001b5aee0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b5aea8 sp=0xc001b5ad58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b5afe0 sp=0xc001b5aea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b5afe8 sp=0xc001b5afe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 852 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d47d60 sp=0xc001d47d40 pc=0x4777ae +runtime.chanrecv(0xc0014374a0, 0xc001d47e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001d47dd8 sp=0xc001d47d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001d47e00 sp=0xc001d47dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001d47fe0 sp=0xc001d47e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d47fe8 sp=0xc001d47fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 825 [select]: +runtime.gopark(0xc001bddf70?, 0x2?, 0xb4?, 0x1b?, 0xc001bddebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bddd20 sp=0xc001bddd00 pc=0x4777ae +runtime.selectgo(0xc001bddf70, 0xc001bddeb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bdde70 sp=0xc001bddd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d39e0, {0x5b426e0, 0xc001aba9b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001bddfa0 sp=0xc001bdde70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001bddfe0 sp=0xc001bddfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bddfe8 sp=0xc001bddfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 822 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 826 [select]: +runtime.gopark(0xc001bdbf70?, 0x2?, 0xb4?, 0x1b?, 0xc001bdbebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bdbd20 sp=0xc001bdbd00 pc=0x4777ae +runtime.selectgo(0xc001bdbf70, 0xc001bdbeb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bdbe70 sp=0xc001bdbd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d3a60, {0x5b426e0, 0xc001abaa00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001bdbfa0 sp=0xc001bdbe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001bdbfe0 sp=0xc001bdbfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bdbfe8 sp=0xc001bdbfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 822 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1056 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0x28?, 0x8a?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001984e18 sp=0xc001984df8 pc=0x4777ae +runtime.chanrecv(0xc00235d680, 0xc001984ed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001984e90 sp=0xc001984e18 pc=0x441325 +runtime.chanrecv2(0xc001658a28?, 0x4be3ae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc001984eb8 sp=0xc001984e90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc001658a00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc001984fc0 sp=0xc001984eb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc001984fe0 sp=0xc001984fc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001984fe8 sp=0xc001984fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 823 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 1148 [IO wait]: +runtime.gopark(0xc0029d3650?, 0x4d1aca?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029d35e0 sp=0xc0029d35c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0029d3618 sp=0xc0029d35e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494240, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0029d3638 sp=0xc0029d3618 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0027d4720, 0xc000dee000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0029d3680 sp=0xc0029d3638 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0027d4700, {0xc000dee000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0029d3770 sp=0xc0029d3680 pc=0x50e5a5 +net.(*netFD).Read(0xc0027d4700, {0xc000dee000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0029d37d0 sp=0xc0029d3770 pc=0x89342b +net.(*conn).Read(0xc000bdfcc8, {0xc000dee000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0029d3858 sp=0xc0029d37d0 pc=0x8b176d +net.(*TCPConn).Read(0xc000bdfcc8, {0xc000dee000, 0x1000, 0x1000}) + :1 +0x54 fp=0xc0029d38a0 sp=0xc0029d3858 pc=0x8d72f4 +net/http.(*connReader).Read(0xc0021ba1b0, {0xc000dee000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:791 +0x2b5 fp=0xc0029d3938 sp=0xc0029d38a0 pc=0xa68115 +bufio.(*Reader).fill(0xc000d64360) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:113 +0x29a fp=0xc0029d39c8 sp=0xc0029d3938 pc=0x5f863a +bufio.(*Reader).Peek(0xc000d64360, 0x4) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:151 +0xc7 fp=0xc0029d3a50 sp=0xc0029d39c8 pc=0x5f88e7 +net/http.(*conn).serve(0xc001172360, {0x5b426a8, 0xc0020083f0}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2044 +0xe7c fp=0xc0029d3fa0 sp=0xc0029d3a50 pc=0xa7221c +net/http.(*Server).Serve.func3() + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3086 +0x50 fp=0xc0029d3fe0 sp=0xc0029d3fa0 pc=0xa78af0 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029d3fe8 sp=0xc0029d3fe0 pc=0x4ad421 +created by net/http.(*Server).Serve in goroutine 1161 + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3086 +0x80d + +goroutine 829 [select]: +runtime.gopark(0xc001a0bf70?, 0x2?, 0xb4?, 0x1b?, 0xc001a0bebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a0bd20 sp=0xc001a0bd00 pc=0x4777ae +runtime.selectgo(0xc001a0bf70, 0xc001a0beb8, 0x4aead9?, 0x0, 0x215774d?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a0be70 sp=0xc001a0bd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d3ee0, {0x5b426e0, 0xc001abac30}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001a0bfa0 sp=0xc001a0be70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001a0bfe0 sp=0xc001a0bfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a0bfe8 sp=0xc001a0bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 823 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 830 [select]: +runtime.gopark(0xc001b0df70?, 0x2?, 0xb4?, 0x1b?, 0xc001b0debc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b0dd20 sp=0xc001b0dd00 pc=0x4777ae +runtime.selectgo(0xc001b0df70, 0xc001b0deb8, 0x4aead9?, 0x0, 0x215774d?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b0de70 sp=0xc001b0dd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0009d3f60, {0x5b426e0, 0xc001abac80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001b0dfa0 sp=0xc001b0de70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001b0dfe0 sp=0xc001b0dfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b0dfe8 sp=0xc001b0dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 823 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 906 [chan receive]: +runtime.gopark(0xc00106e5f0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e3def8 sp=0xc001e3ded8 pc=0x4777ae +runtime.chanrecv(0xc001143560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001e3df70 sp=0xc001e3def8 pc=0x441325 +runtime.chanrecv1(0xc00106e5f0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001e3df98 sp=0xc001e3df70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001e3dfe0 sp=0xc001e3df98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e3dfe8 sp=0xc001e3dfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 904 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 971 [chan receive]: +runtime.gopark(0xc000b3d0b0?, 0x28?, 0x0?, 0x0?, 0x460fed5?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d4aef8 sp=0xc001d4aed8 pc=0x4777ae +runtime.chanrecv(0xc001143560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001d4af70 sp=0xc001d4aef8 pc=0x441325 +runtime.chanrecv1(0xc00106e5f0?, 0x47e975?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001d4af98 sp=0xc001d4af70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001d4afe0 sp=0xc001d4af98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d4afe8 sp=0xc001d4afe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 969 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 864 [select]: +runtime.gopark(0xc001e3bf28?, 0x2?, 0xb4?, 0x1b?, 0xc001e3bebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e3bd30 sp=0xc001e3bd10 pc=0x4777ae +runtime.selectgo(0xc001e3bf28, 0xc001e3beb8, 0x0?, 0x0, 0xc001e3bf20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e3be80 sp=0xc001e3bd30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc0019255f0, 0xc00149e420, 0xc0018f0c40) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001e3bf58 sp=0xc001e3be80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI(0xc0019255f0, {0x5b6f770?, 0xc0010c0e70}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:185 +0x10a fp=0xc001e3bf98 sp=0xc001e3bf58 pc=0x21a7e0a +github.com/rancher/opni/pkg/management.NewServer.func1.2() + /home/ec2-user/opni/pkg/management/server.go:156 +0x62 fp=0xc001e3bfe0 sp=0xc001e3bf98 pc=0x231c0c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e3bfe8 sp=0xc001e3bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 860 + /home/ec2-user/opni/pkg/management/server.go:156 +0x137 + +goroutine 907 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002063978 sp=0xc002063958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0020639b0 sp=0xc002063978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b948, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0020639d0 sp=0xc0020639b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0017cfca0, 0xc002063a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002063a18 sp=0xc0020639d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0017cfc80) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc002063b08 sp=0xc002063a18 pc=0x513f65 +net.(*netFD).accept(0xc0017cfc80) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc002063c00 sp=0xc002063b08 pc=0x8967be +net.(*UnixListener).accept(0xc001be2480) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc002063c40 sp=0xc002063c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc001be2480) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc002063cb0 sp=0xc002063c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc00127d4c0) + :1 +0x44 fp=0xc002063ce8 sp=0xc002063cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc001ebfe00, {0x5b2c6a0, 0xc00127d4c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc002063f18 sp=0xc002063ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc002063f68 sp=0xc002063f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc00127d880?, 0xc0018f11a0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc002063fa8 sp=0xc002063f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc002063fe0 sp=0xc002063fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002063fe8 sp=0xc002063fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 904 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 762 [select]: +runtime.gopark(0xc001b6af20?, 0x2?, 0xb4?, 0x1b?, 0xc001b6aeb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b6ad28 sp=0xc001b6ad08 pc=0x4777ae +runtime.selectgo(0xc001b6af20, 0xc001b6aeb0, 0xc001b6af98?, 0x0, 0xc001b6af18?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b6ae78 sp=0xc001b6ad28 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc001924a80, 0xc00114dcd0, 0xc00114dce0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001b6af50 sp=0xc001b6ae78 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore(0xc001924a80, {0x5b43370?, 0xc00114dcc0}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:214 +0x1a8 fp=0xc001b6af98 sp=0xc001b6af50 pc=0x21a84a8 +github.com/rancher/opni/pkg/gateway.NewGateway.func2.1() + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x62 fp=0xc001b6afe0 sp=0xc001b6af98 pc=0x22b18c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b6afe8 sp=0xc001b6afe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.NewGateway.func2 in goroutine 814 + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x3cc + +goroutine 880 [select]: +runtime.gopark(0xc0030c1e70?, 0x2?, 0x0?, 0x0?, 0xc0030c1df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0030c1c58 sp=0xc0030c1c38 pc=0x4777ae +runtime.selectgo(0xc0030c1e70, 0xc0030c1df0, 0xc002520690?, 0x0, 0xc0017cae00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0030c1da8 sp=0xc0030c1c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001abbe00, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0030c1ea0 sp=0xc0030c1da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002520690) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0030c1f58 sp=0xc0030c1ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0030c1fe0 sp=0xc0030c1f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0030c1fe8 sp=0xc0030c1fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1045 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 844 [select]: +runtime.gopark(0xc001b68fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001b68f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b68de8 sp=0xc001b68dc8 pc=0x4777ae +runtime.selectgo(0xc001b68fb0, 0xc001b68f70, 0xc000081788?, 0x0, 0xc0010880c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b68f38 sp=0xc001b68de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001b68fe0 sp=0xc001b68f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b68fe8 sp=0xc001b68fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 845 [select]: +runtime.gopark(0xc001b67fb0?, 0x2?, 0xb0?, 0x7d?, 0xc001b67ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b67d58 sp=0xc001b67d38 pc=0x4777ae +runtime.selectgo(0xc001b67fb0, 0xc001b67ee0, 0x4aead9?, 0x0, 0x2106cfa?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b67ea8 sp=0xc001b67d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001b67fe0 sp=0xc001b67ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b67fe8 sp=0xc001b67fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 846 [chan receive]: +runtime.gopark(0xc001b66ce8?, 0x11775cf?, 0xd0?, 0x6e?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b66d60 sp=0xc001b66d40 pc=0x4777ae +runtime.chanrecv(0xc001088c60, 0xc001b66e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b66dd8 sp=0xc001b66d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001b66e00 sp=0xc001b66dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001b66fe0 sp=0xc001b66e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b66fe8 sp=0xc001b66fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 764 [select]: +runtime.gopark(0xc00154d448?, 0x2?, 0xd0?, 0xd2?, 0xc00154d3fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00154d278 sp=0xc00154d258 pc=0x4777ae +runtime.selectgo(0xc00154d448, 0xc00154d3f8, 0xc0015113b8?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00154d3c8 sp=0xc00154d278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008a3200) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc00154d478 sp=0xc00154d3c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc00184b110, {0x42219c0?, 0xc001c16420}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc00154d6d8 sp=0xc00154d478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0008a3074?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc00154d738 sp=0xc00154d6d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0008a2fc0, 0xc00154d8f0, 0xc00154d8c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc00154d7e0 sp=0xc00154d738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008a2fc0, {0x42219c0, 0xc001c16420}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc00154d920 sp=0xc00154d7e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc001c164e0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc001c163f0}, {0x42219c0, 0xc001c16420}, 0x0?, {0xc001f0f380, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc00154d9c0 sp=0xc00154d920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc001fefae0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc001c163f0}, {0x42219c0, 0xc001c16420}, 0xc001838400, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc00154dd30 sp=0xc00154d9c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc001838400, {0x5b426e0, 0xc001fefae0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc001c163f0}, {0x42219c0, 0xc001c16420}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc00154de58 sp=0xc00154dd30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseKeyValueStore(0xc00158bb70, {0x5b426e0, 0xc001fefae0}, 0xc001707750?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:69 +0xee fp=0xc00154dee8 sp=0xc00154de58 pc=0x21ab96e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore.func2(0x3) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:219 +0xf2 fp=0xc00154df58 sp=0xc00154dee8 pc=0x21a85d2 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc00154dfe0 sp=0xc00154df58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00154dfe8 sp=0xc00154dfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 762 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 831 [select]: +runtime.gopark(0xc001d79f70?, 0x2?, 0xb4?, 0x1b?, 0xc001d79ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d79d20 sp=0xc001d79d00 pc=0x4777ae +runtime.selectgo(0xc001d79f70, 0xc001d79eb8, 0x4aead9?, 0x0, 0x215774d?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001d79e70 sp=0xc001d79d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d861e0, {0x5b426e0, 0xc001abad70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001d79fa0 sp=0xc001d79e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001d79fe0 sp=0xc001d79fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d79fe8 sp=0xc001d79fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 824 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 832 [select]: +runtime.gopark(0xc001a05f70?, 0x2?, 0xb4?, 0x1b?, 0xc001a05ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001a05d20 sp=0xc001a05d00 pc=0x4777ae +runtime.selectgo(0xc001a05f70, 0xc001a05eb8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001a05e70 sp=0xc001a05d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d86240, {0x5b426e0, 0xc001abadc0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001a05fa0 sp=0xc001a05e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001a05fe0 sp=0xc001a05fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001a05fe8 sp=0xc001a05fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 824 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 899 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xc0?, 0xd3?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d77cd0 sp=0xc001d77cb0 pc=0x4777ae +runtime.chanrecv(0xc001a3b860, 0xc001d77de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001d77d48 sp=0xc001d77cd0 pc=0x441325 +runtime.chanrecv1(0xc001d77df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001d77d70 sp=0xc001d77d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001d77f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001d77e00 sp=0xc001d77d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc0011f24b0, 0x420c60?, 0xc001be2270) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001d77fa0 sp=0xc001d77e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001d77fe0 sp=0xc001d77fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d77fe8 sp=0xc001d77fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 864 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 847 [select]: +runtime.gopark(0xc001fbe890?, 0x2?, 0x98?, 0xe7?, 0xc001fbe83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fbe6a8 sp=0xc001fbe688 pc=0x4777ae +runtime.selectgo(0xc001fbe890, 0xc001fbe838, 0x7f429a45e200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001fbe7f8 sp=0xc001fbe6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00128db30, {0xc001b402e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001fbe8c0 sp=0xc001fbe7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00128db30, {0xc001b402e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001fbe958 sp=0xc001fbe8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001925a10, {0xc001b402e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001fbe9a0 sp=0xc001fbe958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001925a10}, {0xc001b402e0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001fbe9e8 sp=0xc001fbe9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001196900, {0xc001b402e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001fbea58 sp=0xc001fbe9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001b402d0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001fbeb28 sp=0xc001fbea58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001b09e00?, 0xc001196900, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001fbec18 sp=0xc001fbeb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001b402d0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001b6e1e0}, 0xc0010e4480?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001fbecd0 sp=0xc001fbec18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001b9c690, {0x43dcf60?, 0xc001b6e1e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001fbef08 sp=0xc001fbecd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001b40420, {0x43dcf60, 0xc001b6e1e0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001fbeff0 sp=0xc001fbef08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc000b30f00, {0x43dcf60, 0xc001b6e1e0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001fbf028 sp=0xc001fbeff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc0018be7b0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001fbf068 sp=0xc001fbf028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc0018be7b0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001fbf0a8 sp=0xc001fbf068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc0018be7d0) + :1 +0x46 fp=0xc001fbf0e0 sp=0xc001fbf0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc0018be7d0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001fbf320 sp=0xc001fbf0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc0018be7b0}) + :1 +0xaf fp=0xc001fbf370 sp=0xc001fbf320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc000b30f00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001fbf3b0 sp=0xc001fbf370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001b40420}, 0xc000126fc0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001fbf400 sp=0xc001fbf3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001b40420}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001fbf490 sp=0xc001fbf400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9c690}, 0xc000126fc0, 0xc001b09400) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001fbf7d0 sp=0xc001fbf490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9c690}, 0x3f252a0?, 0xc0018be6e0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001fbf848 sp=0xc001fbf7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc001196900, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001fbfdb0 sp=0xc001fbf848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc001196900, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001fbff28 sp=0xc001fbfdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001fbffe0 sp=0xc001fbff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fbffe8 sp=0xc001fbffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1155 [chan receive]: +runtime.gopark(0x3fa3c40?, 0xc002578d20?, 0x0?, 0x0?, 0xc00213df00?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00213df08 sp=0xc00213dee8 pc=0x4777ae +runtime.chanrecv(0xc002578d20, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00213df80 sp=0xc00213df08 pc=0x441325 +runtime.chanrecv1(0xc0024638b0?, 0xc001ebfc20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00213dfa8 sp=0xc00213df80 pc=0x440f72 +github.com/nats-io/nats%2ego.(*js).subscribe.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:1817 +0x4a fp=0xc00213dfe0 sp=0xc00213dfa8 pc=0x1f4624a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00213dfe8 sp=0xc00213dfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*js).subscribe in goroutine 1133 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:1816 +0x2c85 + +goroutine 766 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e5b978 sp=0xc001e5b958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001e5b9b0 sp=0xc001e5b978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163c010, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001e5b9d0 sp=0xc001e5b9b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182da20, 0xc001e5ba01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001e5ba18 sp=0xc001e5b9d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00182da00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001e5bb08 sp=0xc001e5ba18 pc=0x513f65 +net.(*netFD).accept(0xc00182da00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001e5bc00 sp=0xc001e5bb08 pc=0x8967be +net.(*UnixListener).accept(0xc001c166c0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001e5bc40 sp=0xc001e5bc00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc001c166c0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001e5bcb0 sp=0xc001e5bc40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc001d4c120) + :1 +0x44 fp=0xc001e5bce8 sp=0xc001e5bcb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc0018525a0, {0x5b2c6a0, 0xc001d4c120}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001e5bf18 sp=0xc001e5bce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc001e5bf68 sp=0xc001e5bf18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc001d4c280?, 0xc00114df70?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001e5bfa8 sp=0xc001e5bf68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001e5bfe0 sp=0xc001e5bfa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e5bfe8 sp=0xc001e5bfe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 763 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 767 [select]: +runtime.gopark(0xc001b6bf38?, 0x2?, 0x0?, 0x0?, 0xc001b6bf14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b6bd90 sp=0xc001b6bd70 pc=0x4777ae +runtime.selectgo(0xc001b6bf38, 0xc001b6bf10, 0xc001b6bf18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b6bee0 sp=0xc001b6bd90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc001b6bf68 sp=0xc001b6bee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc000dc2078?, 0xc00114df90?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001b6bfa8 sp=0xc001b6bf68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001b6bfe0 sp=0xc001b6bfa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b6bfe8 sp=0xc001b6bfe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 763 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 871 [IO wait]: +runtime.gopark(0xc001fc4fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fc4f98 sp=0xc001fc4f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001fc4fd0 sp=0xc001fc4f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163bf18, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001fc4ff0 sp=0xc001fc4fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001ab06a0, 0xc0025ce000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001fc5038 sp=0xc001fc4ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001ab0680, {0xc0025ce000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001fc5128 sp=0xc001fc5038 pc=0x50e5a5 +net.(*netFD).Read(0xc001ab0680, {0xc0025ce000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001fc5188 sp=0xc001fc5128 pc=0x89342b +net.(*conn).Read(0xc000b79f20, {0xc0025ce000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001fc5210 sp=0xc001fc5188 pc=0x8b176d +net.(*UnixConn).Read(0xc000b79f20, {0xc0025ce000, 0x4000, 0x4000}) + :1 +0x54 fp=0xc001fc5258 sp=0xc001fc5210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc000dc3770, {0xc0025ce000, 0x4000, 0x4000}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001fc52b8 sp=0xc001fc5258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00189f7a8, {0x5b0f6c0, 0xc000dc3770}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001fc5320 sp=0xc001fc52b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00189f500, {0x5b0f840?, 0xc000b79f20}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001fc5390 sp=0xc001fc5320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00189f500, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001fc59c0 sp=0xc001fc5390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00189f500, {0xc002200000, 0x8000, 0xc002339720?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001fc5aa8 sp=0xc001fc59c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d86780, {0xc002200000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001fc5af0 sp=0xc001fc5aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc001de6060, {0xc0011d59a0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001fc5bb0 sp=0xc001fc5af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001de6060}, {0xc0011d59a0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001fc5bf8 sp=0xc001fc5bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d59a0, 0x9, 0x9}, {0x5b0f0a0, 0xc001de6060}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001fc5c98 sp=0xc001fc5bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d5960) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001fc5dd0 sp=0xc001fc5c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001df0480, 0x1157434?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc001fc5fb0 sp=0xc001fc5dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc001fc5fe0 sp=0xc001fc5fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fc5fe8 sp=0xc001fc5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 833 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 768 [chan receive]: +runtime.gopark(0xc001ded5e4?, 0x4be6c0?, 0xe0?, 0xd5?, 0x4be720?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001eeedd0 sp=0xc001eeedb0 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001eeee48 sp=0xc001eeedd0 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0xc00237c720?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001eeee70 sp=0xc001eeee48 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/gateway.(*Plugin).UseKeyValueStore(0xc000379180, {0x5b43f90?, 0xc001ab6550}) + /home/ec2-user/opni/plugins/metrics/pkg/gateway/system.go:91 +0x610 fp=0xc001eeefd0 sp=0xc001eeee70 pc=0x3db4bf0 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseKeyValueStore(0xc001160340, {0x4b05a9?, 0xc001e5d0f0?}, 0xc001c16a80) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:123 +0x18d fp=0xc001eef098 sp=0xc001eeefd0 pc=0x21a6c2d +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler.func1({0x5b426a8, 0xc001c16b40}, {0x4278de0?, 0xc001c16a80}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:187 +0x83 fp=0xc001eef0f0 sp=0xc001eef098 pc=0x21ac823 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc001c16b40}, {0x4278de0, 0xc001c16a80}, 0xc001d4c360, 0xc000dc21e0) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001eef148 sp=0xc001eef0f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc001c16b40}, {0x4278de0, 0xc001c16a80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc001eef1e8 sp=0xc001eef148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc001c16a50}, {0x4278de0, 0xc001c16a80}, 0xc001d4c360, 0xc0019f1a00) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc001eef6b8 sp=0xc001eef1e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc001c16a50}, {0x4278de0, 0xc001c16a80}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001eef740 sp=0xc001eef6b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler({0x4278ea0?, 0xc001160340}, {0x5b426a8, 0xc001c16a50}, 0xc002013260, 0xc001ded8a0) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:189 +0x1c4 fp=0xc001eef7d0 sp=0xc001eef740 pc=0x21ac684 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc0008a3320, 0xc0010e57d0, 0x771ecd0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001eefdb0 sp=0xc001eef7d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc0008a3320, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001eeff28 sp=0xc001eefdb0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001eeffe0 sp=0xc001eeff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001eeffe8 sp=0xc001eeffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1015 [select]: +runtime.gopark(0xc001ef0f70?, 0x4?, 0xa9?, 0x5?, 0xc001ef0dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef0c00 sp=0xc001ef0be0 pc=0x4777ae +runtime.selectgo(0xc001ef0f70, 0xc001ef0dd0, 0x9000000000000000?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ef0d50 sp=0xc001ef0c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002135a00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001ef0fc0 sp=0xc001ef0d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001ef0fe0 sp=0xc001ef0fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef0fe8 sp=0xc001ef0fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 966 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 970 [select]: +runtime.gopark(0xc00211b448?, 0x2?, 0xd0?, 0xb2?, 0xc00211b3fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00211b278 sp=0xc00211b258 pc=0x4777ae +runtime.selectgo(0xc00211b448, 0xc00211b3f8, 0xc00158b938?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00211b3c8 sp=0xc00211b278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001440b40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc00211b478 sp=0xc00211b3c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001d51040, {0x42219c0?, 0xc002008d20}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc00211b6d8 sp=0xc00211b478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0014409b4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc00211b738 sp=0xc00211b6d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001440900, 0xc00211b8f0, 0xc00211b8c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc00211b7e0 sp=0xc00211b738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001440900, {0x42219c0, 0xc002008d20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc00211b920 sp=0xc00211b7e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc002008de0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc002008cf0}, {0x42219c0, 0xc002008d20}, 0x0?, {0xc000ee7320, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc00211b9c0 sp=0xc00211b920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc00106e5f0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc002008cf0}, {0x42219c0, 0xc002008d20}, 0xc0018aa800, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc00211bd30 sp=0xc00211b9c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc0018aa800, {0x5b426e0, 0xc00106e5f0}, {0x4661dbf, 0x1f}, {0x4278de0, 0xc002008cf0}, {0x42219c0, 0xc002008d20}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc00211be58 sp=0xc00211bd30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseKeyValueStore(0xc0018be2e0, {0x5b426e0, 0xc00106e5f0}, 0x0?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:69 +0xee fp=0xc00211bee8 sp=0xc00211be58 pc=0x21ab96e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore.func2(0x3) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:219 +0xf2 fp=0xc00211bf58 sp=0xc00211bee8 pc=0x21a85d2 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc00211bfe0 sp=0xc00211bf58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00211bfe8 sp=0xc00211bfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 968 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 1190 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00210a830 sp=0xc00210a810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00210a868 sp=0xc00210a830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494528, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00210a888 sp=0xc00210a868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001508420, 0xc001824000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00210a8d0 sp=0xc00210a888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001508400, {0xc001824000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00210a9c0 sp=0xc00210a8d0 pc=0x50e5a5 +net.(*netFD).Read(0xc001508400, {0xc001824000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00210aa20 sp=0xc00210a9c0 pc=0x89342b +net.(*conn).Read(0xc001ce3b58, {0xc001824000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00210aaa8 sp=0xc00210aa20 pc=0x8b176d +net.(*TCPConn).Read(0xc001ce3b58, {0xc001824000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc00210aaf0 sp=0xc00210aaa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc0014ac1e0, {0xc0014f8660, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00210abb0 sp=0xc00210aaf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0014ac1e0}, {0xc0014f8660, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00210abf8 sp=0xc00210abb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0014f8660, 0x9, 0x9}, {0x5b0f0a0, 0xc0014ac1e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00210ac98 sp=0xc00210abf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0014f8620) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00210add0 sp=0xc00210ac98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000776240, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc00210afb0 sp=0xc00210add0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc00210afe0 sp=0xc00210afb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00210afe8 sp=0xc00210afe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1073 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 956 [chan receive]: +runtime.gopark(0x400a060?, 0xc001e2fcc0?, 0xe3?, 0x45?, 0x7?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029ddb28 sp=0xc0029ddb08 pc=0x4777ae +runtime.chanrecv(0xc002080e40, 0xc0020f3cb8, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0029ddba0 sp=0xc0029ddb28 pc=0x441325 +runtime.chanrecv1(0xc001ff69c0?, 0x771d460?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0029ddbc8 sp=0xc0029ddba0 pc=0x440f72 +github.com/rancher/opni/pkg/gateway.(*StreamServer).OnPluginLoad.func1() + /home/ec2-user/opni/pkg/gateway/stream.go:285 +0x8eb fp=0xc0029ddfe0 sp=0xc0029ddbc8 pc=0x22bfbeb +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029ddfe8 sp=0xc0029ddfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.(*StreamServer).OnPluginLoad in goroutine 934 + /home/ec2-user/opni/pkg/gateway/stream.go:255 +0xa52 + +goroutine 769 [select]: +runtime.gopark(0xc001fb5f70?, 0x2?, 0xb4?, 0x1b?, 0xc001fb5ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fb5d20 sp=0xc001fb5d00 pc=0x4777ae +runtime.selectgo(0xc001fb5f70, 0xc001fb5eb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001fb5e70 sp=0xc001fb5d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d4c540, {0x5b426e0, 0xc001c64780}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001fb5fa0 sp=0xc001fb5e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001fb5fe0 sp=0xc001fb5fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fb5fe8 sp=0xc001fb5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 768 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 882 [select]: +runtime.gopark(0xc001e5df70?, 0x2?, 0xb4?, 0x1b?, 0xc001e5debc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e5dd20 sp=0xc001e5dd00 pc=0x4777ae +runtime.selectgo(0xc001e5df70, 0xc001e5deb8, 0x4b3745?, 0x0, 0xc001fefb48?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e5de70 sp=0xc001e5dd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d4c5a0, {0x5b426e0, 0xc001c647d0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001e5dfa0 sp=0xc001e5de70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001e5dfe0 sp=0xc001e5dfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e5dfe8 sp=0xc001e5dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 768 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 865 [select]: +runtime.gopark(0xc001e3cf28?, 0x2?, 0xb4?, 0x1b?, 0xc001e3cebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e3cd30 sp=0xc001e3cd10 pc=0x4777ae +runtime.selectgo(0xc001e3cf28, 0xc001e3ceb8, 0x22c21ff?, 0x0, 0xc001e3cf20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e3ce80 sp=0xc001e3cd30 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc0019255f0, 0xc00149e510, 0xc0018f0e10) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001e3cf58 sp=0xc001e3ce80 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer(0xc0019255f0, {0x5b21b70?, 0xc0017e1740}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:198 +0x10a fp=0xc001e3cf98 sp=0xc001e3cf58 pc=0x21a810a +github.com/rancher/opni/pkg/management.NewServer.func1.3() + /home/ec2-user/opni/pkg/management/server.go:158 +0x62 fp=0xc001e3cfe0 sp=0xc001e3cf98 pc=0x231c022 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e3cfe8 sp=0xc001e3cfe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/management.NewServer.func1 in goroutine 860 + /home/ec2-user/opni/pkg/management/server.go:158 +0x245 + +goroutine 1174 [select]: +runtime.gopark(0xc002c92828?, 0x2?, 0x8?, 0x0?, 0xc002c9252c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c92300 sp=0xc002c922e0 pc=0x4777ae +runtime.selectgo(0xc002c92828, 0xc002c92528, 0xc002c7f8d0?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c92450 sp=0xc002c92300 pc=0x48916b +github.com/rancher/opni/pkg/gateway.(*StreamServer).Connect(0xc0020917c0, {0x5b508a0, 0xc002857580}) + /home/ec2-user/opni/pkg/gateway/stream.go:181 +0x1e05 fp=0xc002c92e80 sp=0xc002c92450 pc=0x22bcf85 +github.com/rancher/opni/pkg/apis/stream/v1._Stream_Connect_Handler({0x432f2e0?, 0xc0020917c0}, {0x5b4ab30?, 0xc0027cc1e0}) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:121 +0xb6 fp=0xc002c92ec0 sp=0xc002c92e80 pc=0x2003ed6 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x432f2e0, 0xc0020917c0}, {0x5b4a6f0, 0xc002488960}, 0xc001e46060, 0x47ee1d0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002c93200 sp=0xc002c92ec0 pc=0x12b0e02 +google.golang.org/grpc.getChainStreamHandler.func1({0x432f2e0, 0xc0020917c0}, {0x5b4a6f0, 0xc002488960}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002c93290 sp=0xc002c93200 pc=0x117fe75 +github.com/rancher/opni/pkg/gateway.NewGateway.NewLastKnownDetailsApplier.func16({0x432f2e0, 0xc0020917c0}, {0x5b4a6f0?, 0xc002488960}, 0xc001e46060?, 0xc001d7d9c0) + /home/ec2-user/opni/pkg/gateway/versions.go:54 +0x5e8 fp=0xc002c93388 sp=0xc002c93290 pc=0x22af588 +google.golang.org/grpc.getChainStreamHandler.func1({0x432f2e0, 0xc0020917c0}, {0x5b4a6f0, 0xc002488960}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002c93418 sp=0xc002c93388 pc=0x117fe75 +github.com/rancher/opni/pkg/gateway.NewGateway.(*UpdateServer).StreamServerInterceptor.func15({0x432f2e0, 0xc0020917c0}, {0x5b4a6f0, 0xc002bf8da0}, 0xc001e46060?, 0xc001d7d880) + /home/ec2-user/opni/pkg/update/server.go:150 +0xbac fp=0xc002c93580 sp=0xc002c93418 pc=0x22b02ac +google.golang.org/grpc.getChainStreamHandler.func1({0x432f2e0, 0xc0020917c0}, {0x5b4a6f0, 0xc002bf8da0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002c93610 sp=0xc002c93580 pc=0x117fe75 +github.com/rancher/opni/pkg/gateway.NewGateway.StreamServerInterceptor.func9({0x432f2e0, 0xc0020917c0}, {0x5b492f8, 0xc001670d20}, 0xc001e46060?, 0xc00236da40) + /home/ec2-user/opni/pkg/auth/cluster/middleware.go:19 +0x199 fp=0xc002c93680 sp=0xc002c93610 pc=0x22b08f9 +google.golang.org/grpc.getChainStreamHandler.func1({0x432f2e0, 0xc0020917c0}, {0x5b492f8, 0xc001670d20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002c93710 sp=0xc002c93680 pc=0x117fe75 +github.com/rancher/opni/pkg/gateway.NewGateway.(*RatelimiterInterceptor).StreamServerInterceptor.func14({0x432f2e0, 0xc0020917c0}, {0x5b492f8, 0xc001670d20}, 0xc001e46060, 0xc00236da00) + /home/ec2-user/opni/pkg/gateway/ratelimit.go:65 +0x7e fp=0xc002c93778 sp=0xc002c93710 pc=0x22b03be +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x432f2e0, 0xc0020917c0}, {0x5b492f8, 0xc001670d20}, 0x3f252a0?, 0xc0026f2720?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002c937f0 sp=0xc002c93778 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00170e000, {0x5b54e00, 0xc002515040}, 0xc001d8ad80, 0xc002172ae0, 0x770c800, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002c93d58 sp=0xc002c937f0 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00170e000, {0x5b54e00, 0xc002515040}, 0xc001d8ad80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002c93ed0 sp=0xc002c93d58 pc=0x1184745 +google.golang.org/grpc.(*Server).handleSingleStream(0x1?, 0xc002bf8d80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:604 +0x165 fp=0xc002c93f88 sp=0xc002c93ed0 pc=0x1174725 +google.golang.org/grpc.(*Server).serverWorker(0xc00170e000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:597 +0x38 fp=0xc002c93fc0 sp=0xc002c93f88 pc=0x1174458 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc002c93fe0 sp=0xc002c93fc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c93fe8 sp=0xc002c93fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 1088 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 1318 [chan receive]: +runtime.gopark(0xc0019f9e30?, 0x4c0446?, 0x0?, 0x0?, 0x4aead9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0019f9e68 sp=0xc0019f9e48 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0019f9ee0 sp=0xc0019f9e68 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0019f9f08 sp=0xc0019f9ee0 pc=0x440f72 +github.com/rancher/opni/plugins/alerting/test.(*TestEnvAlertingClusterDriver).StartAlertingBackendServer.func1() + /home/ec2-user/opni/plugins/alerting/test/test_drivers.go:384 +0x5c fp=0xc0019f9f50 sp=0xc0019f9f08 pc=0x398719c +github.com/rancher/opni/pkg/util/waitctx.permissive.Go.func1() + /home/ec2-user/opni/pkg/util/waitctx/context.go:222 +0x93 fp=0xc0019f9fe0 sp=0xc0019f9f50 pc=0x178e3d3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0019f9fe8 sp=0xc0019f9fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/util/waitctx.permissive.Go in goroutine 1250 + /home/ec2-user/opni/pkg/util/waitctx/context.go:220 +0xeb + +goroutine 1081 [select]: +runtime.gopark(0xc001e64890?, 0x2?, 0xe0?, 0x46?, 0xc001e6483c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e646a8 sp=0xc001e64688 pc=0x4777ae +runtime.selectgo(0xc001e64890, 0xc001e64838, 0x7f429a45d300?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e647f8 sp=0xc001e646a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00255caa0, {0xc00253f5d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001e648c0 sp=0xc001e647f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00255caa0, {0xc00253f5d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001e64958 sp=0xc001e648c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00253f530, {0xc00253f5d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001e649a0 sp=0xc001e64958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc00253f530}, {0xc00253f5d0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001e649e8 sp=0xc001e649a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000c04b40, {0xc00253f5d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001e64a58 sp=0xc001e649e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc00253f5c0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001e64b28 sp=0xc001e64a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc000fe6680?, 0xc000c04b40, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001e64c18 sp=0xc001e64b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc00253f5c0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001662f50}, 0x1?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001e64cd0 sp=0xc001e64c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001aae000, {0x43dcf60?, 0xc001662f50}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001e64f08 sp=0xc001e64cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc00253f710, {0x43dcf60, 0xc001662f50}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001e64ff0 sp=0xc001e64f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001d87e60, {0x43dcf60, 0xc001662f50}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001e65028 sp=0xc001e64ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc002576540) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001e65068 sp=0xc001e65028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc002576540?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001e650a8 sp=0xc001e65068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc002576550) + :1 +0x46 fp=0xc001e650e0 sp=0xc001e650a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc002576550}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001e65320 sp=0xc001e650e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc002576540}) + :1 +0xaf fp=0xc001e65370 sp=0xc001e65320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc001d87e60}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001e653b0 sp=0xc001e65370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc00253f710}, 0xc0012e5518?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001e65400 sp=0xc001e653b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc00253f710}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001e65490 sp=0xc001e65400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001aae000}, 0xc0012e5518, 0xc002570380) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001e657d0 sp=0xc001e65490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001aae000}, 0x3f252a0?, 0xc002576470?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001e65848 sp=0xc001e657d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc000c04b40, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001e65db0 sp=0xc001e65848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc000c04b40, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001e65f28 sp=0xc001e65db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001e65fe0 sp=0xc001e65f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e65fe8 sp=0xc001e65fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 900 [select]: +runtime.gopark(0xc001b49448?, 0x2?, 0xd0?, 0x92?, 0xc001b493fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b49278 sp=0xc001b49258 pc=0x4777ae +runtime.selectgo(0xc001b49448, 0xc001b493f8, 0xc00158b938?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b493c8 sp=0xc001b49278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00123c240) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc001b49478 sp=0xc001b493c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bd0c30, {0x42219c0?, 0xc001be24e0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc001b496d8 sp=0xc001b49478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000926654?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc001b49738 sp=0xc001b496d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0009265a0, 0xc001b498f0, 0xc001b498c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc001b497e0 sp=0xc001b49738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0009265a0, {0x42219c0, 0xc001be24e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc001b49920 sp=0xc001b497e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc001be25a0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc001be24b0}, {0x42219c0, 0xc001be24e0}, 0x0?, {0xc000ee7320, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc001b499c0 sp=0xc001b49920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc00106e5f0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc001be24b0}, {0x42219c0, 0xc001be24e0}, 0xc0018aa800, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc001b49d30 sp=0xc001b499c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc0018aa800, {0x5b426e0, 0xc00106e5f0}, {0x4661da0, 0x1f}, {0x4278de0, 0xc001be24b0}, {0x42219c0, 0xc001be24e0}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc001b49e58 sp=0xc001b49d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseManagementAPI(0xc0018be2e0, {0x5b426e0, 0xc00106e5f0}, 0xc000086750?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:51 +0xee fp=0xc001b49ee8 sp=0xc001b49e58 pc=0x21ab66e +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeManagementAPI.func2(0x1) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:190 +0xf2 fp=0xc001b49f58 sp=0xc001b49ee8 pc=0x21a7f32 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc001b49fe0 sp=0xc001b49f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b49fe8 sp=0xc001b49fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 864 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 1005 [select]: +runtime.gopark(0xc001633e48?, 0x2?, 0x0?, 0x0?, 0xc001633dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001633c30 sp=0xc001633c10 pc=0x4777ae +runtime.selectgo(0xc001633e48, 0xc001633dc8, 0xc001633dd8?, 0x0, 0xc00189fc00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001633d80 sp=0xc001633c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0021e8640, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001633e78 sp=0xc001633d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00243d810) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001633f30 sp=0xc001633e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc001633fe0 sp=0xc001633f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001633fe8 sp=0xc001633fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 873 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 969 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0xd5?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020efcd0 sp=0xc0020efcb0 pc=0x4777ae +runtime.chanrecv(0xc0020477a0, 0xc0020efde0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0020efd48 sp=0xc0020efcd0 pc=0x441325 +runtime.chanrecv1(0xc0020efdf0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0020efd70 sp=0xc0020efd48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc0020eff30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc0020efe00 sp=0xc0020efd70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc0011f24b0, 0x420c60?, 0xc002008c90) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc0020effa0 sp=0xc0020efe00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc0020effe0 sp=0xc0020effa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020effe8 sp=0xc0020effe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 968 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 901 [chan receive]: +runtime.gopark(0x22c2125?, 0xc000086f50?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d48ef8 sp=0xc001d48ed8 pc=0x4777ae +runtime.chanrecv(0xc001143560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001d48f70 sp=0xc001d48ef8 pc=0x441325 +runtime.chanrecv1(0xc00106e5f0?, 0x22c2050?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001d48f98 sp=0xc001d48f70 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1.1() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:254 +0x5e fp=0xc001d48fe0 sp=0xc001d48f98 pc=0x21a975e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d48fe8 sp=0xc001d48fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func1 in goroutine 899 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:253 +0x54b + +goroutine 902 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fc1978 sp=0xc001fc1958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001fc19b0 sp=0xc001fc1978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163ba40, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001fc19d0 sp=0xc001fc19b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0017cfb20, 0xc001fc1a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001fc1a18 sp=0xc001fc19d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0017cfb00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001fc1b08 sp=0xc001fc1a18 pc=0x513f65 +net.(*netFD).accept(0xc0017cfb00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001fc1c00 sp=0xc001fc1b08 pc=0x8967be +net.(*UnixListener).accept(0xc001be22a0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc001fc1c40 sp=0xc001fc1c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc001be22a0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc001fc1cb0 sp=0xc001fc1c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc00127d1c0) + :1 +0x44 fp=0xc001fc1ce8 sp=0xc001fc1cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc001ebfc20, {0x5b2c6a0, 0xc00127d1c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc001fc1f18 sp=0xc001fc1ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc001fc1f68 sp=0xc001fc1f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc00127d340?, 0xc0018f0dc0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001fc1fa8 sp=0xc001fc1f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001fc1fe0 sp=0xc001fc1fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fc1fe8 sp=0xc001fc1fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 899 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 903 [select]: +runtime.gopark(0xc001982f38?, 0x2?, 0x69?, 0x97?, 0xc001982f14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001982d90 sp=0xc001982d70 pc=0x4777ae +runtime.selectgo(0xc001982f38, 0xc001982f10, 0xc001982f18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001982ee0 sp=0xc001982d90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc001982f68 sp=0xc001982ee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc00149e4f8?, 0xc0018f0de0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001982fa8 sp=0xc001982f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001982fe0 sp=0xc001982fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001982fe8 sp=0xc001982fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 899 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 904 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x40?, 0xd9?, 0x20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fb3cd0 sp=0xc001fb3cb0 pc=0x4777ae +runtime.chanrecv(0xc001a3be60, 0xc001fb3de0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001fb3d48 sp=0xc001fb3cd0 pc=0x441325 +runtime.chanrecv1(0xc001fb3df0?, 0x2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001fb3d70 sp=0xc001fb3d48 pc=0x440f72 +github.com/oklog/run.(*Group).Run(0xc001fb3f30) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:43 +0x25a fp=0xc001fb3e00 sp=0xc001fb3d70 pc=0x2106b1a +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe(0xc0011f24b0, 0x2?, 0xc001be2450) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:357 +0x854 fp=0xc001fb3fa0 sp=0xc001fb3e00 pc=0x2156934 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func3() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x4d fp=0xc001fb3fe0 sp=0xc001fb3fa0 pc=0x21a90ed +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fb3fe8 sp=0xc001fb3fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 865 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:246 +0x365 + +goroutine 905 [select]: +runtime.gopark(0xc002031448?, 0x2?, 0xd0?, 0x12?, 0xc0020313fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002031278 sp=0xc002031258 pc=0x4777ae +runtime.selectgo(0xc002031448, 0xc0020313f8, 0xc00158b938?, 0x0, 0x4515e8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020313c8 sp=0xc002031278 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00123c6c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc002031478 sp=0xc0020313c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bd0d00, {0x42219c0?, 0xc001be27e0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0020316d8 sp=0xc002031478 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00123c414?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002031738 sp=0xc0020316d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00123c360, 0xc0020318f0, 0xc0020318c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x198 fp=0xc0020317e0 sp=0xc002031738 pc=0x118ea78 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00123c360, {0x42219c0, 0xc001be27e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002031920 sp=0xc0020317e0 pc=0x119077e +google.golang.org/grpc.invoke({0x5b426a8, 0xc001be28a0}, {0x4673e80, 0x23}, {0x4278de0, 0xc001be27b0}, {0x42219c0, 0xc001be27e0}, 0x0?, {0xc000ee7320, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0x13a fp=0xc0020319c0 sp=0xc002031920 pc=0x115873a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryClientInterceptor.func1({0x5b426e0, 0xc00106e5f0}, {0x4673e80, 0x23}, {0x4278de0, 0xc001be27b0}, {0x42219c0, 0xc001be27e0}, 0xc0018aa800, 0x47ecba0, ...) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:100 +0x584 fp=0xc002031d30 sp=0xc0020319c0 pc=0x12abd84 +google.golang.org/grpc.(*ClientConn).Invoke(0xc0018aa800, {0x5b426e0, 0xc00106e5f0}, {0x4673e80, 0x23}, {0x4278de0, 0xc001be27b0}, {0x42219c0, 0xc001be27e0}, {0x0, ...}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:40 +0x337 fp=0xc002031e58 sp=0xc002031d30 pc=0x11583d7 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemClient).UseNodeManagerClient(0xc0018be2e0, {0x5b426e0, 0xc00106e5f0}, 0x0?, {0x0, 0x0, 0x0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:60 +0xee fp=0xc002031ee8 sp=0xc002031e58 pc=0x21ab7ee +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeNodeManagerServer.func2(0x2) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:203 +0xf2 fp=0xc002031f58 sp=0xc002031ee8 pc=0x21a8232 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi.func2() + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:263 +0x7a fp=0xc002031fe0 sp=0xc002031f58 pc=0x21a8ffa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002031fe8 sp=0xc002031fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi in goroutine 865 + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:261 +0x445 + +goroutine 1003 [select]: +runtime.gopark(0xc001587e70?, 0x2?, 0x0?, 0x0?, 0xc001587df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001587c58 sp=0xc001587c38 pc=0x4777ae +runtime.selectgo(0xc001587e70, 0xc001587df0, 0xc001587e00?, 0x0, 0xc001969180?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001587da8 sp=0xc001587c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0021e8230, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001587ea0 sp=0xc001587da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001b3ff10) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001587f58 sp=0xc001587ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc001587fe0 sp=0xc001587f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001587fe8 sp=0xc001587fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 927 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 914 [select]: +runtime.gopark(0xc0020f6890?, 0x2?, 0x98?, 0x67?, 0xc0020f683c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020f66a8 sp=0xc0020f6688 pc=0x4777ae +runtime.selectgo(0xc0020f6890, 0xc0020f6838, 0x7f429a458200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020f67f8 sp=0xc0020f66a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001b6e370, {0xc001c17960, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0020f68c0 sp=0xc0020f67f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001b6e370, {0xc001c17960, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0020f6958 sp=0xc0020f68c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001b40720, {0xc001c17960, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0020f69a0 sp=0xc0020f6958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001b40720}, {0xc001c17960, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0020f69e8 sp=0xc0020f69a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001197440, {0xc001c17960, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0020f6a58 sp=0xc0020f69e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001c17950, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0020f6b28 sp=0xc0020f6a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001d7ce80?, 0xc001197440, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0020f6c18 sp=0xc0020f6b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001c17950, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001c65540}, 0xc001266cc0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0020f6cd0 sp=0xc0020f6c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001344d20, {0x43dcf60?, 0xc001c65540}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0020f6f08 sp=0xc0020f6cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001c17aa0, {0x43dcf60, 0xc001c65540}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0020f6ff0 sp=0xc0020f6f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001d4cbe0, {0x43dcf60, 0xc001c65540}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0020f7028 sp=0xc0020f6ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc001ab6cf0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0020f7068 sp=0xc0020f7028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc001ab6cf0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0020f70a8 sp=0xc0020f7068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc001ab6d00) + :1 +0x46 fp=0xc0020f70e0 sp=0xc0020f70a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc001ab6d00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0020f7320 sp=0xc0020f70e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc001ab6cf0}) + :1 +0xaf fp=0xc0020f7370 sp=0xc0020f7320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc001d4cbe0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0020f73b0 sp=0xc0020f7370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001c17aa0}, 0xc000dc2588?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0020f7400 sp=0xc0020f73b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc001c17aa0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0020f7490 sp=0xc0020f7400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001344d20}, 0xc000dc2588, 0xc001d7c280) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0020f77d0 sp=0xc0020f7490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001344d20}, 0x3f252a0?, 0xc001ab6c20?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0020f7848 sp=0xc0020f77d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001197440, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0020f7db0 sp=0xc0020f7848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001197440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0020f7f28 sp=0xc0020f7db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0020f7fe0 sp=0xc0020f7f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020f7fe8 sp=0xc0020f7fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1388 [select]: +runtime.gopark(0xc002161fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002161f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002161de8 sp=0xc002161dc8 pc=0x4777ae +runtime.selectgo(0xc002161fb0, 0xc002161f70, 0xc002161fd0?, 0x0, 0xc002714a80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002161f38 sp=0xc002161de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002161fe0 sp=0xc002161f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002161fe8 sp=0xc002161fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1361 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1228 [select]: +runtime.gopark(0xc00151df10?, 0x3?, 0xb4?, 0x1b?, 0xc00151de92?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00151dca8 sp=0xc00151dc88 pc=0x4777ae +runtime.selectgo(0xc00151df10, 0xc00151de8c, 0x5b126c0?, 0x0, 0x1?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00151ddf8 sp=0xc00151dca8 pc=0x48916b +google.golang.org/grpc.(*addrConn).resetTransport(0xc001f12f00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:1368 +0x3ef fp=0xc00151df50 sp=0xc00151ddf8 pc=0x116334f +google.golang.org/grpc.(*addrConn).connect(0xc001f12f00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:993 +0x23b fp=0xc00151dfc0 sp=0xc00151df50 pc=0x116033b +google.golang.org/grpc.(*acBalancerWrapper).Connect.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0x34 fp=0xc00151dfe0 sp=0xc00151dfc0 pc=0x1157434 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00151dfe8 sp=0xc00151dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*acBalancerWrapper).Connect in goroutine 1037 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0xa7 + +goroutine 908 [select]: +runtime.gopark(0xc001e3ef38?, 0x2?, 0x0?, 0x0?, 0xc001e3ef14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e3ed90 sp=0xc001e3ed70 pc=0x4777ae +runtime.selectgo(0xc001e3ef38, 0xc001e3ef10, 0xc001e3ef18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e3eee0 sp=0xc001e3ed90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc001e3ef68 sp=0xc001e3eee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc00149e780?, 0xc0018f11c0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001e3efa8 sp=0xc001e3ef68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001e3efe0 sp=0xc001e3efa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e3efe8 sp=0xc001e3efe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 904 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 1320 [select]: +runtime.gopark(0xc002c30480?, 0x3?, 0xa0?, 0xea?, 0xc002bfae3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bef1d0 sp=0xc001bef1b0 pc=0x4777ae +runtime.selectgo(0xc002c30480, 0xc002bfae34, 0x0?, 0x0, 0xc0026f2460?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bef320 sp=0xc001bef1d0 pc=0x48916b +reflect.rselect({0xc001bef508, 0x3, 0x7f4241693c48?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:589 +0x345 fp=0xc001bef3d0 sp=0xc001bef320 pc=0x4a8f05 +reflect.Select({0xc002686700, 0x3, 0x2?}) + /home/ec2-user/sdk/go1.21.0/src/reflect/value.go:3104 +0xb8a fp=0xc001bef598 sp=0xc001bef3d0 pc=0x57c44a +github.com/rancher/opni/pkg/util.WaitAll({0x5b426e0, 0xc002c2e550}, 0xc0026f2320, {0xc001bef990, 0x2, 0x2}) + /home/ec2-user/opni/pkg/util/servers.go:51 +0x3ea fp=0xc001bef718 sp=0xc001bef598 pc=0x176ae8a +github.com/rancher/opni/pkg/agent/v2.(*Agent).ListenAndServe(0xc0025ee340, {0x5b426e0, 0xc001fee820}) + /home/ec2-user/opni/pkg/agent/v2/agent.go:449 +0x12d3 fp=0xc001befd10 sp=0xc001bef718 pc=0x21bea33 +github.com/rancher/opni/pkg/test.(*Environment).StartAgent.func3() + /home/ec2-user/opni/pkg/test/environment.go:1976 +0x6ea fp=0xc001beff50 sp=0xc001befd10 pc=0x24c8b6a +github.com/rancher/opni/pkg/util/waitctx.permissive.Go.func1() + /home/ec2-user/opni/pkg/util/waitctx/context.go:222 +0x93 fp=0xc001beffe0 sp=0xc001beff50 pc=0x178e3d3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001beffe8 sp=0xc001beffe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/util/waitctx.permissive.Go in goroutine 609 + /home/ec2-user/opni/pkg/util/waitctx/context.go:220 +0xeb + +goroutine 998 [select]: +runtime.gopark(0xc0022f2890?, 0x2?, 0x98?, 0x27?, 0xc0022f283c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0022f26a8 sp=0xc0022f2688 pc=0x4777ae +runtime.selectgo(0xc0022f2890, 0xc0022f2838, 0x7f429a45ca00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0022f27f8 sp=0xc0022f26a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001b6f6d0, {0xc0021bad30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0022f28c0 sp=0xc0022f27f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001b6f6d0, {0xc0021bad30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0022f2958 sp=0xc0022f28c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0021ba0c0, {0xc0021bad30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0022f29a0 sp=0xc0022f2958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0021ba0c0}, {0xc0021bad30, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0022f29e8 sp=0xc0022f29a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00146aa20, {0xc0021bad30, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0022f2a58 sp=0xc0022f29e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0021bad20, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0022f2b28 sp=0xc0022f2a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0021e6380?, 0xc00146aa20, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0022f2c18 sp=0xc0022f2b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0021bad20, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0021e80a0}, 0xc0010e4480?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0022f2cd0 sp=0xc0022f2c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001b9ca50, {0x43dcf60?, 0xc0021e80a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0022f2f08 sp=0xc0022f2cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0021bae70, {0x43dcf60, 0xc0021e80a0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0022f2ff0 sp=0xc0022f2f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0021805a0, {0x43dcf60, 0xc0021e80a0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0022f3028 sp=0xc0022f2ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc001d2c390) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0022f3068 sp=0xc0022f3028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc001d2c390?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0022f30a8 sp=0xc0022f3068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc001d2c3a0) + :1 +0x46 fp=0xc0022f30e0 sp=0xc0022f30a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc001d2c3a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0022f3320 sp=0xc0022f30e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc001d2c390}) + :1 +0xaf fp=0xc0022f3370 sp=0xc0022f3320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc0021805a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0022f33b0 sp=0xc0022f3370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc0021bae70}, 0xc001e46a38?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0022f3400 sp=0xc0022f33b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc0021bae70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0022f3490 sp=0xc0022f3400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9ca50}, 0xc001e46a38, 0xc001e41980) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0022f37d0 sp=0xc0022f3490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9ca50}, 0x3f252a0?, 0xc001d2c2c0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0022f3848 sp=0xc0022f37d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc00146aa20, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0022f3db0 sp=0xc0022f3848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc00146aa20, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0022f3f28 sp=0xc0022f3db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0022f3fe0 sp=0xc0022f3f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0022f3fe8 sp=0xc0022f3fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 886 [chan receive]: +runtime.gopark(0x4b05a9?, 0xc001f36e68?, 0x6d?, 0x2d?, 0xc000a72d68?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f36dd8 sp=0xc001f36db8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001f36e50 sp=0xc001f36dd8 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x5b6f168?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001f36e78 sp=0xc001f36e50 pc=0x440f72 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseManagementAPI(0xc00070a000, {0x5b6f168, 0xc001a7f950}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:92 +0x619 fp=0xc001f36fa0 sp=0xc001f36e78 pc=0x396f019 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseManagementAPI(0xc001367cc0, {0x4b05a9?, 0xc0020650f0?}, 0xc001c174a0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:102 +0x30f fp=0xc001f37098 sp=0xc001f36fa0 pc=0x21a640f +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler.func1({0x5b426a8, 0xc001c17560}, {0x4278de0?, 0xc001c174a0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:151 +0x83 fp=0xc001f370f0 sp=0xc001f37098 pc=0x21ac063 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc001c17560}, {0x4278de0, 0xc001c174a0}, 0xc001d4c860, 0xc000dc24b0) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001f37148 sp=0xc001f370f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc001c17560}, {0x4278de0, 0xc001c174a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc001f371e8 sp=0xc001f37148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc001c17470}, {0x4278de0, 0xc001c174a0}, 0xc001d4c860, 0xc0019f1e80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc001f376b8 sp=0xc001f371e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc001c17470}, {0x4278de0, 0xc001c174a0}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001f37740 sp=0xc001f376b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseManagementAPI_Handler({0x4278ea0?, 0xc001367cc0}, {0x5b426a8, 0xc001c17470}, 0xc0020139d0, 0xc000ee6aa0) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:153 +0x1c4 fp=0xc001f377d0 sp=0xc001f37740 pc=0x21abec4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001440000, 0xc001267440, 0x771eca0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001f37db0 sp=0xc001f377d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001440000, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001f37f28 sp=0xc001f37db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001f37fe0 sp=0xc001f37f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f37fe8 sp=0xc001f37fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1016 [IO wait]: +runtime.gopark(0xc002232f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002232f40 sp=0xc002232f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002232f78 sp=0xc002232f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494bf0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002232f98 sp=0xc002232f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001f003a0, 0xc002026000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002232fe0 sp=0xc002232f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001f00380, {0xc002026000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0022330d0 sp=0xc002232fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc001f00380, {0xc002026000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002233130 sp=0xc0022330d0 pc=0x89342b +net.(*conn).Read(0xc001ce2b28, {0xc002026000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0022331b8 sp=0xc002233130 pc=0x8b176d +net.(*UnixConn).Read(0xc001ce2b28, {0xc002026000, 0x240, 0x240}) + :1 +0x54 fp=0xc002233200 sp=0xc0022331b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc000dc2fc0, {0xc002026000, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc002233260 sp=0xc002233200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0015210a8, {0x5b0f6c0, 0xc000dc2fc0}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0022332c8 sp=0xc002233260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001520e00, {0x5b0f840?, 0xc001ce2b28}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc002233338 sp=0xc0022332c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001520e00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc002233968 sp=0xc002233338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001520e00, {0xc00230a000, 0x8000, 0x0?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc002233a50 sp=0xc002233968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001fab3e0, {0xc00230a000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc002233a98 sp=0xc002233a50 pc=0x10969ed +bufio.(*Reader).Read(0xc002308360, {0xc000881b60, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002233b58 sp=0xc002233a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002308360}, {0xc000881b60, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002233ba0 sp=0xc002233b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000881b60, 0x9, 0x9}, {0x5b0f0a0, 0xc002308360}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002233c40 sp=0xc002233ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000881b20) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002233d78 sp=0xc002233c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002135a00, 0x0?, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc002233ec8 sp=0xc002233d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc001ebfe00, {0x5b54e00?, 0xc002135a00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc002233f80 sp=0xc002233ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc002233fe0 sp=0xc002233f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002233fe8 sp=0xc002233fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 966 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 887 [chan receive]: +runtime.gopark(0x4be6c0?, 0xc001266090?, 0x20?, 0xe7?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002112f08 sp=0xc002112ee8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002112f80 sp=0xc002112f08 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x5b11a20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002112fa8 sp=0xc002112f80 pc=0x440f72 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseNodeManagerClient(0xc00070a000, {0x5b11a20, 0xc001a7f4a0}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:222 +0x93 fp=0xc002112fd0 sp=0xc002112fa8 pc=0x3972293 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseNodeManagerClient(0xc001367cc0, {0x4b05a9?, 0xc00205f0f0?}, 0xc001c175f0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:113 +0x18d fp=0xc002113098 sp=0xc002112fd0 pc=0x21a68cd +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler.func1({0x5b426a8, 0xc001c176b0}, {0x4278de0?, 0xc001c175f0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:169 +0x83 fp=0xc0021130f0 sp=0xc002113098 pc=0x21ac443 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc001c176b0}, {0x4278de0, 0xc001c175f0}, 0xc001d4c8e0, 0xc000dc24f8) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc002113148 sp=0xc0021130f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc001c176b0}, {0x4278de0, 0xc001c175f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc0021131e8 sp=0xc002113148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc001c175c0}, {0x4278de0, 0xc001c175f0}, 0xc001d4c8e0, 0xc0019f1f80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc0021136b8 sp=0xc0021131e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc001c175c0}, {0x4278de0, 0xc001c175f0}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc002113740 sp=0xc0021136b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseNodeManagerClient_Handler({0x4278ea0?, 0xc001367cc0}, {0x5b426a8, 0xc001c175c0}, 0xc002013b90, 0xc000ee6aa0) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:171 +0x1c4 fp=0xc0021137d0 sp=0xc002113740 pc=0x21ac2a4 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001440120, 0xc001267440, 0x771ecb8, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc002113db0 sp=0xc0021137d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001440120, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc002113f28 sp=0xc002113db0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002113fe0 sp=0xc002113f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002113fe8 sp=0xc002113fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 931 [select]: +runtime.gopark(0xc0020c5fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0020c5f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020c5de8 sp=0xc0020c5dc8 pc=0x4777ae +runtime.selectgo(0xc0020c5fb0, 0xc0020c5f70, 0x4b05a9?, 0x0, 0x8d0f55?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020c5f38 sp=0xc0020c5de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0020c5fe0 sp=0xc0020c5f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020c5fe8 sp=0xc0020c5fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 888 [select]: +runtime.gopark(0xc0020edf70?, 0x2?, 0xb4?, 0x1b?, 0xc0020edebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020edd20 sp=0xc0020edd00 pc=0x4777ae +runtime.selectgo(0xc0020edf70, 0xc0020edeb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020ede70 sp=0xc0020edd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d4caa0, {0x5b426e0, 0xc001c65180}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0020edfa0 sp=0xc0020ede70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0020edfe0 sp=0xc0020edfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020edfe8 sp=0xc0020edfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 915 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 889 [select]: +runtime.gopark(0xc002061f70?, 0x2?, 0xb4?, 0x1b?, 0xc002061ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002061d20 sp=0xc002061d00 pc=0x4777ae +runtime.selectgo(0xc002061f70, 0xc002061eb8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002061e70 sp=0xc002061d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d4cb00, {0x5b426e0, 0xc001c651d0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc002061fa0 sp=0xc002061e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc002061fe0 sp=0xc002061fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002061fe8 sp=0xc002061fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 915 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 932 [select]: +runtime.gopark(0xc001ef2fb0?, 0x2?, 0xb0?, 0x2d?, 0xc001ef2ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef2d58 sp=0xc001ef2d38 pc=0x4777ae +runtime.selectgo(0xc001ef2fb0, 0xc001ef2ee0, 0x8ab66c?, 0x0, 0x4b0571?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ef2ea8 sp=0xc001ef2d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001ef2fe0 sp=0xc001ef2ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef2fe8 sp=0xc001ef2fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 933 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef1d60 sp=0xc001ef1d40 pc=0x4777ae +runtime.chanrecv(0xc001be4c60, 0xc001ef1e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001ef1dd8 sp=0xc001ef1d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001ef1e00 sp=0xc001ef1dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001ef1fe0 sp=0xc001ef1e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef1fe8 sp=0xc001ef1fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 957 [select]: +runtime.gopark(0xc00210efb0?, 0x2?, 0x0?, 0xee?, 0xc00210ef2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00210eda8 sp=0xc00210ed88 pc=0x4777ae +runtime.selectgo(0xc00210efb0, 0xc00210ef28, 0x0?, 0x0, 0x13?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00210eef8 sp=0xc00210eda8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc00210efe0 sp=0xc00210eef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00210efe8 sp=0xc00210efe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 956 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 935 [select]: +runtime.gopark(0xc0020c1fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0020c1f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020c1de8 sp=0xc0020c1dc8 pc=0x4777ae +runtime.selectgo(0xc0020c1fb0, 0xc0020c1f70, 0xc001bfe2a0?, 0x0, 0xc001be4cc0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020c1f38 sp=0xc0020c1de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0020c1fe0 sp=0xc0020c1f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020c1fe8 sp=0xc0020c1fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 934 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 936 [select]: +runtime.gopark(0xc0020c2fb0?, 0x2?, 0x10?, 0xf1?, 0xc0020c2ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020c2d58 sp=0xc0020c2d38 pc=0x4777ae +runtime.selectgo(0xc0020c2fb0, 0xc0020c2ee0, 0x1?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020c2ea8 sp=0xc0020c2d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0020c2fe0 sp=0xc0020c2ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020c2fe8 sp=0xc0020c2fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 934 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 937 [chan receive]: +runtime.gopark(0xc00036b950?, 0xc00036bc20?, 0xb0?, 0xbc?, 0xc00043a520?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020c3d60 sp=0xc0020c3d40 pc=0x4777ae +runtime.chanrecv(0xc001be4f60, 0xc0020c3e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0020c3dd8 sp=0xc0020c3d60 pc=0x441325 +runtime.chanrecv1(0xc00036afc0?, 0xc00036b710?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0020c3e00 sp=0xc0020c3dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0020c3fe0 sp=0xc0020c3e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020c3fe8 sp=0xc0020c3fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 934 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1345 [chan receive]: +runtime.gopark(0xc002338e18?, 0x7c5b1283c07db?, 0x0?, 0x44?, 0xc000077180?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001488d10 sp=0xc001488cf0 pc=0x4777ae +runtime.chanrecv(0xc00201d500, 0xc001488e40, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001488d88 sp=0xc001488d10 pc=0x441325 +runtime.chanrecv2(0x3b9aca00?, 0x3fa3c40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc001488db0 sp=0xc001488d88 pc=0x440f92 +github.com/hashicorp/go-plugin.pidWait(0x4b05a9?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/process.go:20 +0xdb fp=0xc001488e68 sp=0xc001488db0 pc=0x216033b +github.com/hashicorp/go-plugin.(*Client).reattach.func1(0x0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:853 +0xd8 fp=0xc001488fb8 sp=0xc001488e68 pc=0x21528b8 +github.com/hashicorp/go-plugin.(*Client).reattach.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:862 +0x42 fp=0xc001488fe0 sp=0xc001488fb8 pc=0x21527a2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001488fe8 sp=0xc001488fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*Client).reattach in goroutine 1361 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:846 +0x365 + +goroutine 968 [select]: +runtime.gopark(0xc001ef7f20?, 0x2?, 0xb4?, 0x1b?, 0xc001ef7eb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef7d28 sp=0xc001ef7d08 pc=0x4777ae +runtime.selectgo(0xc001ef7f20, 0xc001ef7eb0, 0xc001f10480?, 0x0, 0xc001ef7f18?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ef7e78 sp=0xc001ef7d28 pc=0x48916b +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).serveSystemApi(0xc0019255f0, 0xc001ab75c0, 0xc001ab75d0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:265 +0x4c9 fp=0xc001ef7f50 sp=0xc001ef7e78 pc=0x21a8ec9 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginHandler).ServeKeyValueStore(0xc0019255f0, {0x5b43370?, 0xc001ab75b0}) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:214 +0x1a8 fp=0xc001ef7f98 sp=0xc001ef7f50 pc=0x21a84a8 +github.com/rancher/opni/pkg/gateway.NewGateway.func2.1() + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x62 fp=0xc001ef7fe0 sp=0xc001ef7f98 pc=0x22b18c2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef7fe8 sp=0xc001ef7fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.NewGateway.func2 in goroutine 861 + /home/ec2-user/opni/pkg/gateway/gateway.go:157 +0x3cc + +goroutine 938 [select]: +runtime.gopark(0xc002116890?, 0x2?, 0x98?, 0x67?, 0xc00211683c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0021166a8 sp=0xc002116688 pc=0x4777ae +runtime.selectgo(0xc002116890, 0xc002116838, 0x7f429a465700?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0021167f8 sp=0xc0021166a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001ed44b0, {0xc001be3db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0021168c0 sp=0xc0021167f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001ed44b0, {0xc001be3db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002116958 sp=0xc0021168c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001be3590, {0xc001be3db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0021169a0 sp=0xc002116958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001be3590}, {0xc001be3db0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0021169e8 sp=0xc0021169a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00123d0e0, {0xc001be3db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002116a58 sp=0xc0021169e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001be3da0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002116b28 sp=0xc002116a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001ff0680?, 0xc00123d0e0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002116c18 sp=0xc002116b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001be3da0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001ed4be0}, 0xc0010e4480?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002116cd0 sp=0xc002116c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001ec2d20, {0x43dcf60?, 0xc001ed4be0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002116f08 sp=0xc002116cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001be3ef0, {0x43dcf60, 0xc001ed4be0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002116ff0 sp=0xc002116f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001faa160, {0x43dcf60, 0xc001ed4be0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002117028 sp=0xc002116ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc0018f1b60) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc002117068 sp=0xc002117028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc0018f1b60?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0021170a8 sp=0xc002117068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc0018f1b70) + :1 +0x46 fp=0xc0021170e0 sp=0xc0021170a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc0018f1b70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc002117320 sp=0xc0021170e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc0018f1b60}) + :1 +0xaf fp=0xc002117370 sp=0xc002117320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc001faa160}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0021173b0 sp=0xc002117370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001be3ef0}, 0xc00149edf8?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002117400 sp=0xc0021173b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc001be3ef0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002117490 sp=0xc002117400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001ec2d20}, 0xc00149edf8, 0xc001badc80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0021177d0 sp=0xc002117490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001ec2d20}, 0x3f252a0?, 0xc0018f1a90?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002117848 sp=0xc0021177d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc00123d0e0, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002117db0 sp=0xc002117848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc00123d0e0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002117f28 sp=0xc002117db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002117fe0 sp=0xc002117f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002117fe8 sp=0xc002117fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 964 [IO wait]: +runtime.gopark(0xc001fb6fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fb6f98 sp=0xc001fb6f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001fb6fd0 sp=0xc001fb6f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163bb38, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001fb6ff0 sp=0xc001fb6fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182dca0, 0xc002014c00?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001fb7038 sp=0xc001fb6ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00182dc80, {0xc002014c00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001fb7128 sp=0xc001fb7038 pc=0x50e5a5 +net.(*netFD).Read(0xc00182dc80, {0xc002014c00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001fb7188 sp=0xc001fb7128 pc=0x89342b +net.(*conn).Read(0xc001ce2978, {0xc002014c00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001fb7210 sp=0xc001fb7188 pc=0x8b176d +net.(*UnixConn).Read(0xc001ce2978, {0xc002014c00, 0x600, 0x600}) + :1 +0x54 fp=0xc001fb7258 sp=0xc001fb7210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc002e63890, {0xc002014c00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001fb72b8 sp=0xc001fb7258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc001520d28, {0x5b0f6c0, 0xc002e63890}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001fb7320 sp=0xc001fb72b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001520a80, {0x5b0f840?, 0xc001ce2978}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001fb7390 sp=0xc001fb7320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001520a80, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001fb79c0 sp=0xc001fb7390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001520a80, {0xc002280000, 0x8000, 0xc002c2b540?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001fb7aa8 sp=0xc001fb79c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d4d000, {0xc002280000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001fb7af0 sp=0xc001fb7aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc00201d7a0, {0xc000978820, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001fb7bb0 sp=0xc001fb7af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00201d7a0}, {0xc000978820, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001fb7bf8 sp=0xc001fb7bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000978820, 0x9, 0x9}, {0x5b0f0a0, 0xc00201d7a0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001fb7c98 sp=0xc001fb7bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0009787e0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001fb7dd0 sp=0xc001fb7c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000f95d40, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc001fb7fb0 sp=0xc001fb7dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc001fb7fe0 sp=0xc001fb7fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fb7fe8 sp=0xc001fb7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 883 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 939 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0xe0?, 0x39?, 0x30?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020faf98 sp=0xc0020faf78 pc=0x4777ae +runtime.chanrecv(0xc001bfeae0, 0xc0020fb128, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0020fb010 sp=0xc0020faf98 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0020fb038 sp=0xc0020fb010 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/stream.(*gatewayStreamExtensionServerImpl).ConnectInternal(0xc0007d1b90, {0x5b50ab0, 0xc0018f18f0}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/stream/plugin_gateway.go:208 +0xa4e fp=0xc0020fb370 sp=0xc0020fb038 pc=0x262088e +github.com/rancher/opni/pkg/plugins/apis/apiextensions._StreamAPIExtension_ConnectInternal_Handler({0x4229080?, 0xc0007d1b90}, {0x5b4a6f0?, 0xc001faa040}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go:284 +0xb6 fp=0xc0020fb3b0 sp=0xc0020fb370 pc=0x218ded6 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x4229080, 0xc0007d1b90}, {0x5b4ab30, 0xc001be38c0}, 0xc00149ec30?, 0x47ee568) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0020fb400 sp=0xc0020fb3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x4229080, 0xc0007d1b90}, {0x5b4ab30, 0xc001be38c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0020fb490 sp=0xc0020fb400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4229080, 0xc0007d1b90}, {0x5b492f8, 0xc001ec2c30}, 0xc00149ec30, 0xc001bad900) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0020fb7d0 sp=0xc0020fb490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x4229080, 0xc0007d1b90}, {0x5b492f8, 0xc001ec2c30}, 0x3f252a0?, 0xc0018f1820?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0020fb848 sp=0xc0020fb7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc00123d320, 0xc0010e4ed0, 0x770c840, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0020fbdb0 sp=0xc0020fb848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc00123d320, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0020fbf28 sp=0xc0020fbdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0020fbfe0 sp=0xc0020fbf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020fbfe8 sp=0xc0020fbfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 940 [select]: +runtime.gopark(0xc00210ffb0?, 0x2?, 0x0?, 0x4e?, 0xc00210ff2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00210fda8 sp=0xc00210fd88 pc=0x4777ae +runtime.selectgo(0xc00210ffb0, 0xc00210ff28, 0x4b0571?, 0x0, 0x11847d3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00210fef8 sp=0xc00210fda8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc00210ffe0 sp=0xc00210fef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00210ffe8 sp=0xc00210ffe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 939 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 941 [select]: +runtime.gopark(0xc0020bef90?, 0x2?, 0xff?, 0xff?, 0xc0020beeac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020bed28 sp=0xc0020bed08 pc=0x4777ae +runtime.selectgo(0xc0020bef90, 0xc0020beea8, 0x5d5276?, 0x0, 0xc001838400?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020bee78 sp=0xc0020bed28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc0015edd40) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc0020befc0 sp=0xc0020bee78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc0020befe0 sp=0xc0020befc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020befe8 sp=0xc0020befe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 939 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 942 [select]: +runtime.gopark(0xc00203df40?, 0x2?, 0xb4?, 0x1b?, 0xc00203decc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029dbd48 sp=0xc0029dbd28 pc=0x4777ae +runtime.selectgo(0xc0029dbf40, 0xc00203dec8, 0xc001be2f00?, 0x0, 0x8ab66c?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029dbe98 sp=0xc0029dbd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc001ed4780}, 0xc0015eddd0, 0x0, 0xc001bfea20, 0xc0018f19b0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc0029dbf70 sp=0xc0029dbe98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc0029dbfe0 sp=0xc0029dbf70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029dbfe8 sp=0xc0029dbfe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 939 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 943 [chan receive]: +runtime.gopark(0xc001ed4550?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020c4e20 sp=0xc0020c4e00 pc=0x4777ae +runtime.chanrecv(0xc001be5080, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0020c4e98 sp=0xc0020c4e20 pc=0x441325 +runtime.chanrecv1(0xc001be3890?, 0x2106cfa?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0020c4ec0 sp=0xc0020c4e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0020c4fe0 sp=0xc0020c4ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020c4fe8 sp=0xc0020c4fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 939 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 944 [chan receive]: +runtime.gopark(0xc001ed4550?, 0x4b05a9?, 0x50?, 0xe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020c0e20 sp=0xc0020c0e00 pc=0x4777ae +runtime.chanrecv(0xc001be5080, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0020c0e98 sp=0xc0020c0e20 pc=0x441325 +runtime.chanrecv1(0xc001be3890?, 0xc001729870?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0020c0ec0 sp=0xc0020c0e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0020c0fe0 sp=0xc0020c0ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020c0fe8 sp=0xc0020c0fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 939 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 945 [select]: +runtime.gopark(0xc001fc3aa8?, 0x2?, 0x67?, 0x6a?, 0xc001fc39c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fc3838 sp=0xc001fc3818 pc=0x4777ae +runtime.selectgo(0xc001fc3aa8, 0xc001fc39c0, 0xc001fc39d0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001fc3988 sp=0xc001fc3838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc00148edc0) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc001fc3ad8 sp=0xc001fc3988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc00123d440, {0x5b426a8, 0xc001be3890}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc001fc3cf8 sp=0xc001fc3ad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc001fc3fe0 sp=0xc001fc3cf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fc3fe8 sp=0xc001fc3fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 939 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 946 [select]: +runtime.gopark(0xc0020e9710?, 0x2?, 0x0?, 0x0?, 0xc0020e96bc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020e9528 sp=0xc0020e9508 pc=0x4777ae +runtime.selectgo(0xc0020e9710, 0xc0020e96b8, 0x7f429a464200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020e9678 sp=0xc0020e9528 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001ed45a0, {0xc001be3780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0020e9740 sp=0xc0020e9678 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001ed45a0, {0xc001be3780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0020e97d8 sp=0xc0020e9740 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001be36e0, {0xc001be3780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0020e9820 sp=0xc0020e97d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001be36e0}, {0xc001be3780, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0020e9868 sp=0xc0020e9820 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00123d320, {0xc001be3780, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0020e98d8 sp=0xc0020e9868 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001be3770, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0020e99a8 sp=0xc0020e98d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc00123d320, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0020e9a98 sp=0xc0020e99a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001be3770, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc0023002a0}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0020e9b50 sp=0xc0020e9a98 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001ec2c30, {0x43f1c20?, 0xc0023002a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0020e9d88 sp=0xc0020e9b50 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc001be38c0, {0x43f1c20, 0xc0023002a0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0020e9e70 sp=0xc0020e9d88 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001faa040, {0x43f1c20, 0xc0023002a0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0020e9ea8 sp=0xc0020e9e70 pc=0x1fd94b2 +github.com/rancher/opni/pkg/plugins/apis/apiextensions.(*streamAPIExtensionConnectInternalServer).Recv(0xc0018f18f0) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go:303 +0x66 fp=0xc0020e9ee8 sp=0xc0020e9ea8 pc=0x218e046 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc0020e9fe0 sp=0xc0020e9ee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020e9fe8 sp=0xc0020e9fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 945 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1038 [select]: +runtime.gopark(0xc001d75f70?, 0x2?, 0xb4?, 0x1b?, 0xc001d75ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d75d20 sp=0xc001d75d00 pc=0x4777ae +runtime.selectgo(0xc001d75f70, 0xc001d75eb8, 0xc001e6b040?, 0x0, 0x440940?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001d75e70 sp=0xc001d75d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0024893e0, {0x5b426e0, 0xc002055180}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001d75fa0 sp=0xc001d75e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001d75fe0 sp=0xc001d75fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d75fe8 sp=0xc001d75fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 823 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 958 [select]: +runtime.gopark(0xc00210cf90?, 0x2?, 0xff?, 0xff?, 0xc00210ceac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00210cd28 sp=0xc00210cd08 pc=0x4777ae +runtime.selectgo(0xc00210cf90, 0xc00210cea8, 0x0?, 0x0, 0x4b0a55?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00210ce78 sp=0xc00210cd28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc000931200) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc00210cfc0 sp=0xc00210ce78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc00210cfe0 sp=0xc00210cfc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00210cfe8 sp=0xc00210cfe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 956 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 920 [select]: +runtime.gopark(0xc002065f70?, 0x2?, 0xb4?, 0x1b?, 0xc002065ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002065d20 sp=0xc002065d00 pc=0x4777ae +runtime.selectgo(0xc002065f70, 0xc002065eb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002065e70 sp=0xc002065d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b314c0, {0x5b426e0, 0xc001b6e910}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc002065fa0 sp=0xc002065e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc002065fe0 sp=0xc002065fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002065fe8 sp=0xc002065fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 887 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 921 [select]: +runtime.gopark(0xc00205ff70?, 0x2?, 0xb4?, 0x1b?, 0xc00205febc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00205fd20 sp=0xc00205fd00 pc=0x4777ae +runtime.selectgo(0xc00205ff70, 0xc00205feb8, 0x2020200a3736202d?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00205fe70 sp=0xc00205fd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b31540, {0x5b426e0, 0xc001b6e960}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc00205ffa0 sp=0xc00205fe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc00205ffe0 sp=0xc00205ffa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00205ffe8 sp=0xc00205ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 887 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1084 [select]: +runtime.gopark(0xc001151080?, 0x3?, 0xf9?, 0x50?, 0xc00146598a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002937810 sp=0xc0029377f0 pc=0x4777ae +runtime.selectgo(0xc001151080, 0xc001465984, 0x0?, 0x0, 0xc0013404e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002937960 sp=0xc002937810 pc=0x48916b +reflect.rselect({0xc002937b48, 0x3, 0x4279b60?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:589 +0x345 fp=0xc002937a10 sp=0xc002937960 pc=0x4a8f05 +reflect.Select({0xc000ded180, 0x3, 0x2?}) + /home/ec2-user/sdk/go1.21.0/src/reflect/value.go:3104 +0xb8a fp=0xc002937bd8 sp=0xc002937a10 pc=0x57c44a +github.com/rancher/opni/pkg/util.WaitAll({0x5b426e0, 0xc0012c2730}, 0xc0013403a0, {0xc002937db8, 0x2, 0x2}) + /home/ec2-user/opni/pkg/util/servers.go:51 +0x3ea fp=0xc002937d58 sp=0xc002937bd8 pc=0x176ae8a +github.com/rancher/opni/pkg/management.(*Server).ListenAndServe(0xc0010c0e70, {0x5b426e0, 0xc001fee820}) + /home/ec2-user/opni/pkg/management/server.go:196 +0x208 fp=0xc002937dd8 sp=0xc002937d58 pc=0x231df08 +github.com/rancher/opni/pkg/test.(*Environment).startGateway.func2(0x4b0571) + /home/ec2-user/opni/pkg/test/environment.go:1699 +0x18c fp=0xc002937f88 sp=0xc002937dd8 pc=0x24c574c +github.com/rancher/opni/pkg/plugins/hooks.onLoadingCompletedHook.Invoke(0xc001ded3a0, 0xc001448330?) + /home/ec2-user/opni/pkg/plugins/hooks/complete.go:10 +0x34 fp=0xc002937fa8 sp=0xc002937f88 pc=0x2182b94 +github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete.func1() + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x51 fp=0xc002937fe0 sp=0xc002937fa8 pc=0x2189711 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002937fe8 sp=0xc002937fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins.(*PluginLoader).Complete in goroutine 609 + /home/ec2-user/opni/pkg/plugins/loader.go:329 +0x138 + +goroutine 952 [select]: +runtime.gopark(0xc002108fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002108f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002108de8 sp=0xc002108dc8 pc=0x4777ae +runtime.selectgo(0xc002108fb0, 0xc002108f70, 0x4ad421?, 0x0, 0x8d0f0a?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002108f38 sp=0xc002108de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002108fe0 sp=0xc002108f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002108fe8 sp=0xc002108fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 717 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 953 [select]: +runtime.gopark(0xc002109fb0?, 0x2?, 0xb0?, 0x9d?, 0xc002109ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002109d58 sp=0xc002109d38 pc=0x4777ae +runtime.selectgo(0xc002109fb0, 0xc002109ee0, 0x1381849?, 0x0, 0xc001729870?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002109ea8 sp=0xc002109d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002109fe0 sp=0xc002109ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002109fe8 sp=0xc002109fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 954 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00210dd60 sp=0xc00210dd40 pc=0x4777ae +runtime.chanrecv(0xc001be5980, 0xc00210de60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00210ddd8 sp=0xc00210dd60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00210de00 sp=0xc00210ddd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc00210dfe0 sp=0xc00210de00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00210dfe8 sp=0xc00210dfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 717 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 922 [select]: +runtime.gopark(0xc001e61f70?, 0x2?, 0xb4?, 0x1b?, 0xc001e61ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e61d20 sp=0xc001e61d00 pc=0x4777ae +runtime.selectgo(0xc001e61f70, 0xc001e61eb8, 0x2020200a3736202d?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e61e70 sp=0xc001e61d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b31bc0, {0x5b426e0, 0xc001b6eaf0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001e61fa0 sp=0xc001e61e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001e61fe0 sp=0xc001e61fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e61fe8 sp=0xc001e61fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 886 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 923 [select]: +runtime.gopark(0xc001e5ff70?, 0x2?, 0xb4?, 0x1b?, 0xc001e5febc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e5fd20 sp=0xc001e5fd00 pc=0x4777ae +runtime.selectgo(0xc001e5ff70, 0xc001e5feb8, 0x233731f?, 0x0, 0x472ddb?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e5fe70 sp=0xc001e5fd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b31ce0, {0x5b426e0, 0xc001b6eb40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001e5ffa0 sp=0xc001e5fe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001e5ffe0 sp=0xc001e5ffa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e5ffe8 sp=0xc001e5ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 886 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1040 [select]: +runtime.gopark(0xc002939f70?, 0x2?, 0xb4?, 0x1b?, 0xc002939ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002939d20 sp=0xc002939d00 pc=0x4777ae +runtime.selectgo(0xc002939f70, 0xc002939eb8, 0x0?, 0x0, 0x4aef8c?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002939e70 sp=0xc002939d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0024896a0, {0x5b426e0, 0xc002055450}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc002939fa0 sp=0xc002939e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc002939fe0 sp=0xc002939fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002939fe8 sp=0xc002939fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 823 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1035 [select]: +runtime.gopark(0xc001e3ff80?, 0x3?, 0xb4?, 0x1b?, 0xc001e3fee2?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e3fd30 sp=0xc001e3fd10 pc=0x4777ae +runtime.selectgo(0xc001e3ff80, 0xc001e3fedc, 0x4aef8c?, 0x0, 0xc001c65640?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e3fe80 sp=0xc001e3fd30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc0022efaa0) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:52 +0x178 fp=0xc001e3ffc0 sp=0xc001e3fe80 pc=0x13b0398 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc001e3ffe0 sp=0xc001e3ffc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e3ffe8 sp=0xc001e3ffe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 1017 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 872 [select]: +runtime.gopark(0xc00224be70?, 0x2?, 0x0?, 0x0?, 0xc00224bdf4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00203dc58 sp=0xc00203dc38 pc=0x4777ae +runtime.selectgo(0xc00203de70, 0xc00224bdf0, 0xc00224be00?, 0x0, 0xc00189f500?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00203dda8 sp=0xc00203dc58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001abb6d0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00203dea0 sp=0xc00203dda8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0018b9420) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00203df58 sp=0xc00203dea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc00203dfe0 sp=0xc00203df58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00203dfe8 sp=0xc00203dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 833 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1055 [IO wait]: +runtime.gopark(0xc00215fbf0?, 0x26?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e1fbd0 sp=0xc002e1fbb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002e1fc08 sp=0xc002e1fbd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494718, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002e1fc28 sp=0xc002e1fc08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0025f8920, 0xc002732000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002e1fc70 sp=0xc002e1fc28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0025f8900, {0xc002732000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc002e1fd60 sp=0xc002e1fc70 pc=0x50e5a5 +net.(*netFD).Read(0xc0025f8900, {0xc002732000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002e1fdc0 sp=0xc002e1fd60 pc=0x89342b +net.(*conn).Read(0xc001d6fda8, {0xc002732000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc002e1fe48 sp=0xc002e1fdc0 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6fda8, {0xc002732000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc002e1fe90 sp=0xc002e1fe48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc0027226c0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc002e1ff10 sp=0xc002e1fe90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc001658a00) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc002e1ffc0 sp=0xc002e1ff10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc002e1ffe0 sp=0xc002e1ffc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e1ffe8 sp=0xc002e1ffe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 823 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 1002 [IO wait]: +runtime.gopark(0xc0021b8fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0021b8f98 sp=0xc0021b8f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0021b8fd0 sp=0xc0021b8f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494de0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0021b8ff0 sp=0xc0021b8fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193baa0, 0xc00042e800?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0021b9038 sp=0xc0021b8ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00193ba80, {0xc00042e800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0021b9128 sp=0xc0021b9038 pc=0x50e5a5 +net.(*netFD).Read(0xc00193ba80, {0xc00042e800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0021b9188 sp=0xc0021b9128 pc=0x89342b +net.(*conn).Read(0xc000991968, {0xc00042e800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0021b9210 sp=0xc0021b9188 pc=0x8b176d +net.(*UnixConn).Read(0xc000991968, {0xc00042e800, 0x3800, 0x3800}) + :1 +0x54 fp=0xc0021b9258 sp=0xc0021b9210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0030c5b48, {0xc00042e800, 0x3800, 0x3800}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0021b92b8 sp=0xc0021b9258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc001969428, {0x5b0f6c0, 0xc0030c5b48}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0021b9320 sp=0xc0021b92b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001969180, {0x5b0f840?, 0xc000991968}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0021b9390 sp=0xc0021b9320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001969180, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0021b99c0 sp=0xc0021b9390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001969180, {0xc0021f4000, 0x8000, 0xc003207360?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0021b9aa8 sp=0xc0021b99c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0021805e0, {0xc0021f4000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0021b9af0 sp=0xc0021b9aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc0021db020, {0xc0019391c0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0021b9bb0 sp=0xc0021b9af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0021db020}, {0xc0019391c0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0021b9bf8 sp=0xc0021b9bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0019391c0, 0x9, 0x9}, {0x5b0f0a0, 0xc0021db020}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0021b9c98 sp=0xc0021b9bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001939180) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0021b9dd0 sp=0xc0021b9c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001c6db00, 0xc001e6bd40?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0021b9fb0 sp=0xc0021b9dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0021b9fe0 sp=0xc0021b9fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0021b9fe8 sp=0xc0021b9fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 927 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1447 [select]: +runtime.gopark(0xc002105fb0?, 0x2?, 0xb0?, 0x5d?, 0xc002105ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002105d58 sp=0xc002105d38 pc=0x4777ae +runtime.selectgo(0xc002105fb0, 0xc002105ee0, 0xe?, 0x0, 0x8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002105ea8 sp=0xc002105d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002105fe0 sp=0xc002105ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002105fe8 sp=0xc002105fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 874 [select]: +runtime.gopark(0xc002243e48?, 0x2?, 0x0?, 0x0?, 0xc002243dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020ffc30 sp=0xc0020ffc10 pc=0x4777ae +runtime.selectgo(0xc0020ffe48, 0xc002243dc8, 0xc002243dd8?, 0x0, 0xc0017ca380?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020ffd80 sp=0xc0020ffc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001abb7c0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0020ffe78 sp=0xc0020ffd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0018b9570) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0020fff30 sp=0xc0020ffe78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0020fffe0 sp=0xc0020fff30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020fffe8 sp=0xc0020fffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 858 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 875 [select]: +runtime.gopark(0xc0017d7f70?, 0x4?, 0x29?, 0x0?, 0xc0017d7dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d7c00 sp=0xc0017d7be0 pc=0x4777ae +runtime.selectgo(0xc0017d7f70, 0xc0017d7dd0, 0xc0017d7f08?, 0x0, 0x5b50310?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017d7d50 sp=0xc0017d7c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001afda00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc0017d7fc0 sp=0xc0017d7d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc0017d7fe0 sp=0xc0017d7fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d7fe8 sp=0xc0017d7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 858 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 876 [IO wait]: +runtime.gopark(0xc001fc6f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fc6f40 sp=0xc001fc6f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001fc6f78 sp=0xc001fc6f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163be20, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001fc6f98 sp=0xc001fc6f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0017cf7a0, 0xc000d40fc0?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001fc6fe0 sp=0xc001fc6f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0017cf780, {0xc000d40fc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001fc70d0 sp=0xc001fc6fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc0017cf780, {0xc000d40fc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001fc7130 sp=0xc001fc70d0 pc=0x89342b +net.(*conn).Read(0xc001d6f4d0, {0xc000d40fc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001fc71b8 sp=0xc001fc7130 pc=0x8b176d +net.(*UnixConn).Read(0xc001d6f4d0, {0xc000d40fc0, 0x240, 0x240}) + :1 +0x54 fp=0xc001fc7200 sp=0xc001fc71b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00217fdd0, {0xc000d40fc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001fc7260 sp=0xc001fc7200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0017ca628, {0x5b0f6c0, 0xc00217fdd0}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001fc72c8 sp=0xc001fc7260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc0017ca380, {0x5b0f840?, 0xc001d6f4d0}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001fc7338 sp=0xc001fc72c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc0017ca380, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001fc7968 sp=0xc001fc7338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc0017ca380, {0xc002218000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001fc7a50 sp=0xc001fc7968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d86840, {0xc002218000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001fc7a98 sp=0xc001fc7a50 pc=0x10969ed +bufio.(*Reader).Read(0xc001de63c0, {0xc0011d5a80, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001fc7b58 sp=0xc001fc7a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001de63c0}, {0xc0011d5a80, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001fc7ba0 sp=0xc001fc7b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d5a80, 0x9, 0x9}, {0x5b0f0a0, 0xc001de63c0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001fc7c40 sp=0xc001fc7ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d5a40) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001fc7d78 sp=0xc001fc7c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001afda00, 0x0?, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001fc7ec8 sp=0xc001fc7d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c780, {0x5b54e00?, 0xc001afda00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001fc7f80 sp=0xc001fc7ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001fc7fe0 sp=0xc001fc7f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fc7fe8 sp=0xc001fc7fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 858 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 959 [select]: +runtime.gopark(0xc00212ff40?, 0x2?, 0xb4?, 0x1b?, 0xc00212fecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029e5d48 sp=0xc0029e5d28 pc=0x4777ae +runtime.selectgo(0xc0029e5f40, 0xc00212fec8, 0xc001be2f00?, 0x0, 0x8ab66c?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029e5e98 sp=0xc0029e5d48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc001ed5450}, 0xc000931290, 0x0, 0xc002080cc0, 0xc001c90ba0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc0029e5f70 sp=0xc0029e5e98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc0029e5fe0 sp=0xc0029e5f70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029e5fe8 sp=0xc0029e5fe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 956 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 960 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc001be4e40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00213ae20 sp=0xc00213ae00 pc=0x4777ae +runtime.chanrecv(0xc001be4e40, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00213ae98 sp=0xc00213ae20 pc=0x441325 +runtime.chanrecv1(0xc001be3350?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00213aec0 sp=0xc00213ae98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc00213afe0 sp=0xc00213aec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00213afe8 sp=0xc00213afe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 956 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 961 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc001be4e40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002163e20 sp=0xc002163e00 pc=0x4777ae +runtime.chanrecv(0xc001be4e40, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002163e98 sp=0xc002163e20 pc=0x441325 +runtime.chanrecv1(0xc001be3350?, 0xc000086ee8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002163ec0 sp=0xc002163e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002163fe0 sp=0xc002163ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002163fe8 sp=0xc002163fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 956 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 978 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc001be4e40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00213ce20 sp=0xc00213ce00 pc=0x4777ae +runtime.chanrecv(0xc001be4e40, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00213ce98 sp=0xc00213ce20 pc=0x441325 +runtime.chanrecv1(0xc001be3350?, 0x43148a0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00213cec0 sp=0xc00213ce98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc00213cfe0 sp=0xc00213cec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00213cfe8 sp=0xc00213cfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 956 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 979 [select]: +runtime.gopark(0xc001d7baa8?, 0x2?, 0x67?, 0x6a?, 0xc001d7b9c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d7b838 sp=0xc001d7b818 pc=0x4777ae +runtime.selectgo(0xc001d7baa8, 0xc001d7b9c0, 0xc001d7b9d0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001d7b988 sp=0xc001d7b838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc00148f0e0) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc001d7bad8 sp=0xc001d7b988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc00123dc20, {0x5b426a8, 0xc001be3350}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc001d7bcf8 sp=0xc001d7bad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc001d7bfe0 sp=0xc001d7bcf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d7bfe8 sp=0xc001d7bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 956 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 980 [select]: +runtime.gopark(0xc0020eb3f8?, 0x2?, 0xc0?, 0x72?, 0xc0020eb37c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020eb1e8 sp=0xc0020eb1c8 pc=0x4777ae +runtime.selectgo(0xc0020eb3f8, 0xc0020eb378, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020eb338 sp=0xc0020eb1e8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc001ed4370, {0xc001be3420, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc0020eb428 sp=0xc0020eb338 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001ed4370, {0xc001be3420, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc0020eb4c0 sp=0xc0020eb428 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001be33e0, {0xc001be3420, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0020eb508 sp=0xc0020eb4c0 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001be33e0}, {0xc001be3420, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0020eb550 sp=0xc0020eb508 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00123cfc0, {0xc001be3420, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0020eb5c0 sp=0xc0020eb550 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc001be3410, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0020eb690 sp=0xc0020eb5c0 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc00123cfc0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0020eb780 sp=0xc0020eb690 pc=0x1171c5c +google.golang.org/grpc.recv(0xc001be3410, {0x7f424347ae80, 0x779c980}, 0x448485?, {0x0, 0x0}, {0x43f1c20, 0xc002300690}, 0x800?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0020eb838 sp=0xc0020eb780 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bd1930, {0x43f1c20?, 0xc002300690}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc0020eba98 sp=0xc0020eb838 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00123ce34?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc0020ebaf8 sp=0xc0020eba98 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00123cd80, 0xc0020ebcb0, 0xc0020ebc80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc0020ebba0 sp=0xc0020ebaf8 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00123cd80, {0x43f1c20, 0xc002300690}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc0020ebce0 sp=0xc0020ebba0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc001bad780, {0x43f1c20, 0xc002300690}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc0020ebea8 sp=0xc0020ebce0 pc=0x12ac5b9 +github.com/rancher/opni/pkg/plugins/apis/apiextensions.(*streamAPIExtensionConnectInternalClient).Recv(0xc0018f16e0) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/apiextensions_grpc.pb.go:249 +0x66 fp=0xc0020ebee8 sp=0xc0020ebea8 pc=0x218dd66 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc0020ebfe0 sp=0xc0020ebee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020ebfe8 sp=0xc0020ebfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 979 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 982 [IO wait]: +runtime.gopark(0xc002234fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002234f98 sp=0xc002234f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002234fd0 sp=0xc002234f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163bd28, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002234ff0 sp=0xc002234fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001ab0920, 0xc002130600?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002235038 sp=0xc002234ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001ab0900, {0xc002130600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc002235128 sp=0xc002235038 pc=0x50e5a5 +net.(*netFD).Read(0xc001ab0900, {0xc002130600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002235188 sp=0xc002235128 pc=0x89342b +net.(*conn).Read(0xc000b79f30, {0xc002130600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc002235210 sp=0xc002235188 pc=0x8b176d +net.(*UnixConn).Read(0xc000b79f30, {0xc002130600, 0x600, 0x600}) + :1 +0x54 fp=0xc002235258 sp=0xc002235210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00217e4c8, {0xc002130600, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0022352b8 sp=0xc002235258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00189fb28, {0x5b0f6c0, 0xc00217e4c8}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc002235320 sp=0xc0022352b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00189f880, {0x5b0f840?, 0xc000b79f30}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc002235390 sp=0xc002235320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00189f880, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0022359c0 sp=0xc002235390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00189f880, {0xc00213e000, 0x8000, 0xc002235b38?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc002235aa8 sp=0xc0022359c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001faa600, {0xc00213e000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc002235af0 sp=0xc002235aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc002081260, {0xc000881700, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002235bb0 sp=0xc002235af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002081260}, {0xc000881700, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002235bf8 sp=0xc002235bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000881700, 0x9, 0x9}, {0x5b0f0a0, 0xc002081260}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002235c98 sp=0xc002235bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0008816c0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002235dd0 sp=0xc002235c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001bfc900, 0xc0020812c0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc002235fb0 sp=0xc002235dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc002235fe0 sp=0xc002235fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002235fe8 sp=0xc002235fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 868 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 965 [select]: +runtime.gopark(0xc003025e70?, 0x2?, 0x0?, 0x0?, 0xc003025df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc003025c58 sp=0xc003025c38 pc=0x4777ae +runtime.selectgo(0xc003025e70, 0xc003025df0, 0xc002006e70?, 0x0, 0xc001520a80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc003025da8 sp=0xc003025c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001c65c20, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc003025ea0 sp=0xc003025da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002006e70) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc003025f58 sp=0xc003025ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc003025fe0 sp=0xc003025f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc003025fe8 sp=0xc003025fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 883 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 983 [select]: +runtime.gopark(0xc00216fe70?, 0x2?, 0x0?, 0x0?, 0xc00216fdf4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e1fc58 sp=0xc001e1fc38 pc=0x4777ae +runtime.selectgo(0xc001e1fe70, 0xc00216fdf0, 0xc00216fe00?, 0x0, 0xc00189f880?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e1fda8 sp=0xc001e1fc58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001ed57c0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001e1fea0 sp=0xc001e1fda8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0017dbc00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001e1ff58 sp=0xc001e1fea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc001e1ffe0 sp=0xc001e1ff58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e1ffe8 sp=0xc001e1ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 868 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1017 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x30?, 0x15?, 0x10?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001eeacf8 sp=0xc001eeacd8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001eead70 sp=0xc001eeacf8 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0x5b55580?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001eead98 sp=0xc001eead70 pc=0x440f72 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).UseKeyValueStore(0xc00070a000, {0x5b43f90?, 0xc00234e480}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/system.go:185 +0xc25 fp=0xc001eeafd0 sp=0xc001eead98 pc=0x3970905 +github.com/rancher/opni/pkg/plugins/apis/system.(*systemPluginClientImpl).UseKeyValueStore(0xc001367cc0, {0x4b05a9?, 0xc0023350f0?}, 0xc0023262a0) + /home/ec2-user/opni/pkg/plugins/apis/system/plugin.go:123 +0x18d fp=0xc001eeb098 sp=0xc001eeafd0 pc=0x21a6c2d +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler.func1({0x5b426a8, 0xc002326360}, {0x4278de0?, 0xc0023262a0}) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:187 +0x83 fp=0xc001eeb0f0 sp=0xc001eeb098 pc=0x21ac823 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.(*GrpcClientTtlCacher).UnaryServerInterceptor.func3({0x5b426a8, 0xc002326360}, {0x4278de0, 0xc0023262a0}, 0xc001fab480, 0xc00217e6a8) + /home/ec2-user/opni/pkg/caching/interceptors.go:263 +0x6a fp=0xc001eeb148 sp=0xc001eeb0f0 pc=0x218a26a +google.golang.org/grpc.getChainUnaryHandler.func1({0x5b426a8, 0xc002326360}, {0x4278de0, 0xc0023262a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1179 +0x123 fp=0xc001eeb1e8 sp=0xc001eeb148 pc=0x117b363 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x5b426a8, 0xc002326270}, {0x4278de0, 0xc0023262a0}, 0xc001fab480, 0xc002170d80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:344 +0x6e9 fp=0xc001eeb6b8 sp=0xc001eeb1e8 pc=0x12af1e9 +google.golang.org/grpc.NewServer.chainUnaryServerInterceptors.chainUnaryInterceptors.func1({0x5b426a8, 0xc002326270}, {0x4278de0, 0xc0023262a0}, 0x4357140?, 0x4b05a9?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1170 +0xd9 fp=0xc001eeb740 sp=0xc001eeb6b8 pc=0x1175b99 +github.com/rancher/opni/pkg/plugins/apis/system._System_UseKeyValueStore_Handler({0x4278ea0?, 0xc001367cc0}, {0x5b426a8, 0xc002326270}, 0xc002300620, 0xc000ee6aa0) + /home/ec2-user/opni/pkg/plugins/apis/system/system_grpc.pb.go:189 +0x1c4 fp=0xc001eeb7d0 sp=0xc001eeb740 pc=0x21ac684 +google.golang.org/grpc.(*Server).processUnaryRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001335440, 0xc001267440, 0x771ecd0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1360 +0x1835 fp=0xc001eebdb0 sp=0xc001eeb7d0 pc=0x117cc15 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001335440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1737 +0xfb3 fp=0xc001eebf28 sp=0xc001eebdb0 pc=0x1184793 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001eebfe0 sp=0xc001eebf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001eebfe8 sp=0xc001eebfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1014 [select]: +runtime.gopark(0xc00232fe48?, 0x2?, 0x0?, 0x0?, 0xc00232fdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002123c30 sp=0xc002123c10 pc=0x4777ae +runtime.selectgo(0xc002123e48, 0xc00232fdc8, 0xc00232fdd8?, 0x0, 0xc001520e00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002123d80 sp=0xc002123c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002167bd0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002123e78 sp=0xc002123d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0023005b0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002123f30 sp=0xc002123e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002123fe0 sp=0xc002123f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002123fe8 sp=0xc002123fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 966 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1454 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e1acd0 sp=0xc001e1acb0 pc=0x4777ae +runtime.chanrecv(0xc002d20240, 0xc001e1af90, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001e1ad48 sp=0xc001e1acd0 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001e1ad70 sp=0xc001e1ad48 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/stream.(*agentStreamExtensionServerImpl).Connect(0xc001e54e40, {0x5b508a0, 0xc002d00490}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/stream/plugin_agent.go:166 +0x1065 fp=0xc001e1b370 sp=0xc001e1ad70 pc=0x261cb65 +github.com/rancher/opni/pkg/apis/stream/v1._Stream_Connect_Handler({0x4229120?, 0xc001e54e40}, {0x5b4a6f0?, 0xc002488b60}) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:121 +0xb6 fp=0xc001e1b3b0 sp=0xc001e1b370 pc=0x2003ed6 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x4229120, 0xc001e54e40}, {0x5b4ab30, 0xc00266d9e0}, 0xc0012e4f18?, 0x47ee1d0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001e1b400 sp=0xc001e1b3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x4229120, 0xc001e54e40}, {0x5b4ab30, 0xc00266d9e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001e1b490 sp=0xc001e1b400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4229120, 0xc001e54e40}, {0x5b492f8, 0xc000b9c2d0}, 0xc0012e4f18, 0xc001d7de40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001e1b7d0 sp=0xc001e1b490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x4229120, 0xc001e54e40}, {0x5b492f8, 0xc000b9c2d0}, 0x3f252a0?, 0xc002d001f0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001e1b848 sp=0xc001e1b7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c58360, 0xc0025cd6b0, 0x770c800, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001e1bdb0 sp=0xc001e1b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c58360, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001e1bf28 sp=0xc001e1bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001e1bfe0 sp=0xc001e1bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e1bfe8 sp=0xc001e1bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1387 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 988 [select]: +runtime.gopark(0xc0017d8fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0017d8f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d8de8 sp=0xc0017d8dc8 pc=0x4777ae +runtime.selectgo(0xc0017d8fb0, 0xc0017d8f70, 0x4ad421?, 0x0, 0x8d0f0a?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017d8f38 sp=0xc0017d8de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0017d8fe0 sp=0xc0017d8f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d8fe8 sp=0xc0017d8fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 989 [select]: +runtime.gopark(0xc00213bfb0?, 0x2?, 0xb0?, 0xbd?, 0xc00213bee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00213bd58 sp=0xc00213bd38 pc=0x4777ae +runtime.selectgo(0xc00213bfb0, 0xc00213bee0, 0xf?, 0x0, 0x8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00213bea8 sp=0xc00213bd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc00213bfe0 sp=0xc00213bea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00213bfe8 sp=0xc00213bfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 990 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002139d60 sp=0xc002139d40 pc=0x4777ae +runtime.chanrecv(0xc002156600, 0xc002139e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002139dd8 sp=0xc002139d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002139e00 sp=0xc002139dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002139fe0 sp=0xc002139e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002139fe8 sp=0xc002139fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 994 [select]: +runtime.gopark(0xc002034890?, 0x2?, 0x98?, 0x47?, 0xc00203483c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020346a8 sp=0xc002034688 pc=0x4777ae +runtime.selectgo(0xc002034890, 0xc002034838, 0x7f429a45f700?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020347f8 sp=0xc0020346a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc001b6f4f0, {0xc0021ba9d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0020348c0 sp=0xc0020347f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001b6f4f0, {0xc0021ba9d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002034958 sp=0xc0020348c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001b41c50, {0xc0021ba9d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0020349a0 sp=0xc002034958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc001b41c50}, {0xc0021ba9d0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0020349e8 sp=0xc0020349a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00146a7e0, {0xc0021ba9d0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002034a58 sp=0xc0020349e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0021ba9c0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002034b28 sp=0xc002034a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001e41840?, 0xc00146a7e0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002034c18 sp=0xc002034b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0021ba9c0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc001b6fc70}, 0xc001266cc0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002034cd0 sp=0xc002034c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001b9c960, {0x43dcf60?, 0xc001b6fc70}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002034f08 sp=0xc002034cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0021bab10, {0x43dcf60, 0xc001b6fc70}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002034ff0 sp=0xc002034f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002180520, {0x43dcf60, 0xc001b6fc70}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002035028 sp=0xc002034ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc001d2c210) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc002035068 sp=0xc002035028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc001d2c210?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0020350a8 sp=0xc002035068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc001d2c220) + :1 +0x46 fp=0xc0020350e0 sp=0xc0020350a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc001266ea0, {0x5b51368, 0xc001d2c220}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc002035320 sp=0xc0020350e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc001510830, {0x5b51038, 0xc001d2c210}) + :1 +0xaf fp=0xc002035370 sp=0xc002035320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc001510830}, {0x5b4a6f0?, 0xc002180520}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0020353b0 sp=0xc002035370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc0021bab10}, 0xc001e46828?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002035400 sp=0xc0020353b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc001510830}, {0x5b4ab30, 0xc0021bab10}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002035490 sp=0xc002035400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001b9c960}, 0xc001e46828, 0xc001e40c40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0020357d0 sp=0xc002035490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc001510830}, {0x5b492f8, 0xc001b9c960}, 0x3f252a0?, 0xc001d2c130?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002035848 sp=0xc0020357d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00146a7e0, 0xc001266f30, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002035db0 sp=0xc002035848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc00146a7e0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002035f28 sp=0xc002035db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002035fe0 sp=0xc002035f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002035fe8 sp=0xc002035fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 995 [select]: +runtime.gopark(0xc0021e5e48?, 0x2?, 0x0?, 0x0?, 0xc0021e5dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001551c30 sp=0xc001551c10 pc=0x4777ae +runtime.selectgo(0xc001551e48, 0xc0021e5dc8, 0xc0021e5dd8?, 0x0, 0xc001968a80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001551d80 sp=0xc001551c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001b6f590, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001551e78 sp=0xc001551d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001b3fe30) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001551f30 sp=0xc001551e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc001551fe0 sp=0xc001551f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001551fe8 sp=0xc001551fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 917 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 996 [select]: +runtime.gopark(0xc0020bff70?, 0x4?, 0x20?, 0xbc?, 0xc0020bfdd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020bfc00 sp=0xc0020bfbe0 pc=0x4777ae +runtime.selectgo(0xc0020bff70, 0xc0020bfdd0, 0xb000000004b0571?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0020bfd50 sp=0xc0020bfc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001e6b6c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc0020bffc0 sp=0xc0020bfd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc0020bffe0 sp=0xc0020bffc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020bffe8 sp=0xc0020bffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 917 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 997 [IO wait]: +runtime.gopark(0xc0021b6f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0021b6f40 sp=0xc0021b6f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0021b6f78 sp=0xc0021b6f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b758, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0021b6f98 sp=0xc0021b6f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193b5a0, 0xc00047f440?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0021b6fe0 sp=0xc0021b6f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00193b580, {0xc00047f440, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0021b70d0 sp=0xc0021b6fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc00193b580, {0xc00047f440, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0021b7130 sp=0xc0021b70d0 pc=0x89342b +net.(*conn).Read(0xc000991868, {0xc00047f440, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0021b71b8 sp=0xc0021b7130 pc=0x8b176d +net.(*UnixConn).Read(0xc000991868, {0xc00047f440, 0x240, 0x240}) + :1 +0x54 fp=0xc0021b7200 sp=0xc0021b71b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00217e120, {0xc00047f440, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0021b7260 sp=0xc0021b7200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc001968d28, {0x5b0f6c0, 0xc00217e120}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0021b72c8 sp=0xc0021b7260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001968a80, {0x5b0f840?, 0xc000991868}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0021b7338 sp=0xc0021b72c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001968a80, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0021b7968 sp=0xc0021b7338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001968a80, {0xc00219a000, 0x8000, 0x0?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0021b7a50 sp=0xc0021b7968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0021802c0, {0xc00219a000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0021b7a98 sp=0xc0021b7a50 pc=0x10969ed +bufio.(*Reader).Read(0xc002194f60, {0xc001939000, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0021b7b58 sp=0xc0021b7a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002194f60}, {0xc001939000, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0021b7ba0 sp=0xc0021b7b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001939000, 0x9, 0x9}, {0x5b0f0a0, 0xc002194f60}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0021b7c40 sp=0xc0021b7ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001938fc0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0021b7d78 sp=0xc0021b7c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001e6b6c0, 0x313231202d000000?, 0xc000f3de70?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc0021b7ec8 sp=0xc0021b7d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c960, {0x5b54e00?, 0xc001e6b6c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc0021b7f80 sp=0xc0021b7ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc0021b7fe0 sp=0xc0021b7f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0021b7fe8 sp=0xc0021b7fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 917 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 999 [select]: +runtime.gopark(0xc00302de48?, 0x2?, 0x0?, 0x0?, 0xc00302ddcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00302dc30 sp=0xc00302dc10 pc=0x4777ae +runtime.selectgo(0xc00302de48, 0xc00302ddc8, 0xc00302ddd8?, 0x0, 0xc001968700?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00302dd80 sp=0xc00302dc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001b6f8b0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00302de78 sp=0xc00302dd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001b3fea0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00302df30 sp=0xc00302de78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc00302dfe0 sp=0xc00302df30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00302dfe8 sp=0xc00302dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 849 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1000 [select]: +runtime.gopark(0xc001ef5f70?, 0x4?, 0x78?, 0xf8?, 0xc001ef5dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef5c00 sp=0xc001ef5be0 pc=0x4777ae +runtime.selectgo(0xc001ef5f70, 0xc001ef5dd0, 0x40a16a0?, 0x0, 0xc001d2c040?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001ef5d50 sp=0xc001ef5c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001e6bd40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001ef5fc0 sp=0xc001ef5d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001ef5fe0 sp=0xc001ef5fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef5fe8 sp=0xc001ef5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 849 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1001 [IO wait]: +runtime.gopark(0xc001f42f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f42f40 sp=0xc001f42f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001f42f78 sp=0xc001f42f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b850, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001f42f98 sp=0xc001f42f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193b420, 0xc00047efc0?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001f42fe0 sp=0xc001f42f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00193b400, {0xc00047efc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001f430d0 sp=0xc001f42fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc00193b400, {0xc00047efc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001f43130 sp=0xc001f430d0 pc=0x89342b +net.(*conn).Read(0xc000991860, {0xc00047efc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001f431b8 sp=0xc001f43130 pc=0x8b176d +net.(*UnixConn).Read(0xc000991860, {0xc00047efc0, 0x240, 0x240}) + :1 +0x54 fp=0xc001f43200 sp=0xc001f431b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0030c4138, {0xc00047efc0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001f43260 sp=0xc001f43200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0019689a8, {0x5b0f6c0, 0xc0030c4138}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc001f432c8 sp=0xc001f43260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001968700, {0x5b0f840?, 0xc000991860}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001f43338 sp=0xc001f432c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001968700, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001f43968 sp=0xc001f43338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001968700, {0xc0021bc000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001f43a50 sp=0xc001f43968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0021803c0, {0xc0021bc000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001f43a98 sp=0xc001f43a50 pc=0x10969ed +bufio.(*Reader).Read(0xc0021953e0, {0xc0019390e0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001f43b58 sp=0xc001f43a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0021953e0}, {0xc0019390e0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001f43ba0 sp=0xc001f43b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0019390e0, 0x9, 0x9}, {0x5b0f0a0, 0xc0021953e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001f43c40 sp=0xc001f43ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0019390a0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001f43d78 sp=0xc001f43c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001e6bd40, 0x4aead9?, 0x4b0571?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001f43ec8 sp=0xc001f43d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc0018525a0, {0x5b54e00?, 0xc001e6bd40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001f43f80 sp=0xc001f43ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001f43fe0 sp=0xc001f43f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f43fe8 sp=0xc001f43fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 849 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 972 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0021b3978 sp=0xc0021b3958 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0021b39b0 sp=0xc0021b3978 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b568, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0021b39d0 sp=0xc0021b39b0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001f00520, 0xc0021b3a01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0021b3a18 sp=0xc0021b39d0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc001f00500) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0021b3b08 sp=0xc0021b3a18 pc=0x513f65 +net.(*netFD).accept(0xc001f00500) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0021b3c00 sp=0xc0021b3b08 pc=0x8967be +net.(*UnixListener).accept(0xc002008cc0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc0021b3c40 sp=0xc0021b3c00 pc=0x8cedfe +net.(*UnixListener).Accept(0xc002008cc0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc0021b3cb0 sp=0xc0021b3c40 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc001d4d260) + :1 +0x44 fp=0xc0021b3ce8 sp=0xc0021b3cb0 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc001852b40, {0x5b2c6a0, 0xc001d4d260}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0021b3f18 sp=0xc0021b3ce8 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:336 +0x47 fp=0xc0021b3f68 sp=0xc0021b3f18 pc=0x2156b67 +github.com/oklog/run.(*Group).Run.func1({0xc001d4d480?, 0xc001ab77d0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc0021b3fa8 sp=0xc0021b3f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc0021b3fe0 sp=0xc0021b3fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0021b3fe8 sp=0xc0021b3fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 969 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 973 [select]: +runtime.gopark(0xc001e39f38?, 0x2?, 0x0?, 0x0?, 0xc001e39f14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e39d90 sp=0xc001e39d70 pc=0x4777ae +runtime.selectgo(0xc001e39f38, 0xc001e39f10, 0xc001e39f18?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e39ee0 sp=0xc001e39d90 pc=0x48916b +github.com/hashicorp/go-plugin.(*GRPCBroker).AcceptAndServe.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:346 +0x9e fp=0xc001e39f68 sp=0xc001e39ee0 pc=0x2156a9e +github.com/oklog/run.(*Group).Run.func1({0xc000dc2cd8?, 0xc001ab77f0?}) + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:38 +0x3a fp=0xc001e39fa8 sp=0xc001e39f68 pc=0x2106cfa +github.com/oklog/run.(*Group).Run.func2() + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:39 +0x50 fp=0xc001e39fe0 sp=0xc001e39fa8 pc=0x2106c90 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e39fe8 sp=0xc001e39fe0 pc=0x4ad421 +created by github.com/oklog/run.(*Group).Run in goroutine 969 + /home/ec2-user/go/pkg/mod/github.com/oklog/run@v1.1.0/group.go:37 +0xae + +goroutine 1011 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0021368d8 sp=0xc0021368b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002136910 sp=0xc0021368d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494a00, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002136930 sp=0xc002136910 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001e9f020, 0xc001ff7701?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002136978 sp=0xc002136930 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc001e9f000) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc002136a68 sp=0xc002136978 pc=0x513f65 +net.(*netFD).accept(0xc001e9f000) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc002136b60 sp=0xc002136a68 pc=0x8967be +net.(*TCPListener).accept(0xc001faac20) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc002136ba8 sp=0xc002136b60 pc=0x8c383e +net.(*TCPListener).Accept(0xc001faac20) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc002136c18 sp=0xc002136ba8 pc=0x8c1ba5 +crypto/tls.(*listener).Accept(0xc00149fec0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/tls.go:66 +0x43 fp=0xc002136c70 sp=0xc002136c18 pc=0x9b2d43 +net/http.(*onceCloseListener).Accept(0xc002173710) + :1 +0x44 fp=0xc002136ca8 sp=0xc002136c70 pc=0xab5544 +net/http.(*Server).Serve(0xc001ec2e10, {0x5b2a5b0, 0xc00149fec0}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc002136ea0 sp=0xc002136ca8 pc=0xa7859f +net/http.Serve(...) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2595 +github.com/gin-gonic/gin.(*Engine).RunListener(0xc0018ac680, {0x5b2a5b0, 0xc00149fec0}) + /home/ec2-user/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/gin.go:565 +0x370 fp=0xc002136f90 sp=0xc002136ea0 pc=0x171e870 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure.func1() + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:72 +0x5c fp=0xc002136fe0 sp=0xc002136f90 pc=0x262a57c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002136fe8 sp=0xc002136fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure in goroutine 1010 + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:71 +0x26d + +goroutine 976 [IO wait]: +runtime.gopark(0xc0021b4fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0021b4f98 sp=0xc0021b4f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0021b4fd0 sp=0xc0021b4f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163b660, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0021b4ff0 sp=0xc0021b4fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00193b8a0, 0xc0022f8000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0021b5038 sp=0xc0021b4ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00193b880, {0xc0022f8000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0021b5128 sp=0xc0021b5038 pc=0x50e5a5 +net.(*netFD).Read(0xc00193b880, {0xc0022f8000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0021b5188 sp=0xc0021b5128 pc=0x89342b +net.(*conn).Read(0xc000991960, {0xc0022f8000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0021b5210 sp=0xc0021b5188 pc=0x8b176d +net.(*UnixConn).Read(0xc000991960, {0xc0022f8000, 0x600, 0x600}) + :1 +0x54 fp=0xc0021b5258 sp=0xc0021b5210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc000dc2f90, {0xc0022f8000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0021b52b8 sp=0xc0021b5258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0019690a8, {0x5b0f6c0, 0xc000dc2f90}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0021b5320 sp=0xc0021b52b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc001968e00, {0x5b0f840?, 0xc000991960}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0021b5390 sp=0xc0021b5320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc001968e00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0021b59c0 sp=0xc0021b5390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc001968e00, {0xc0023da000, 0x8000, 0x4?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0021b5aa8 sp=0xc0021b59c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d4d6a0, {0xc0023da000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0021b5af0 sp=0xc0021b5aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc0022eee40, {0xc000978900, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0021b5bb0 sp=0xc0021b5af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0022eee40}, {0xc000978900, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0021b5bf8 sp=0xc0021b5bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000978900, 0x9, 0x9}, {0x5b0f0a0, 0xc0022eee40}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0021b5c98 sp=0xc0021b5bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0009788c0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0021b5dd0 sp=0xc0021b5c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc002028480, 0xc0022eeea0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0021b5fb0 sp=0xc0021b5dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0021b5fe0 sp=0xc0021b5fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0021b5fe8 sp=0xc0021b5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 924 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 977 [select]: +runtime.gopark(0xc002423e70?, 0x2?, 0x0?, 0x0?, 0xc002423df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001229c58 sp=0xc001229c38 pc=0x4777ae +runtime.selectgo(0xc001229e70, 0xc002423df0, 0xc002423e00?, 0x0, 0xc001968e00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001229da8 sp=0xc001229c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002054460, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc001229ea0 sp=0xc001229da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002007dc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc001229f58 sp=0xc001229ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc001229fe0 sp=0xc001229f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001229fe8 sp=0xc001229fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 924 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1041 [select]: +runtime.gopark(0xc0029e3f70?, 0x2?, 0xb4?, 0x1b?, 0xc0029e3ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029e3d20 sp=0xc0029e3d00 pc=0x4777ae +runtime.selectgo(0xc0029e3f70, 0xc0029e3eb8, 0xc001e6ad00?, 0x0, 0x440940?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029e3e70 sp=0xc0029e3d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002489700, {0x5b426e0, 0xc0020554a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0029e3fa0 sp=0xc0029e3e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0029e3fe0 sp=0xc0029e3fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029e3fe8 sp=0xc0029e3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 823 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1006 [select]: +runtime.gopark(0xc002160f70?, 0x4?, 0x58?, 0xc?, 0xc002160dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002160c00 sp=0xc002160be0 pc=0x4777ae +runtime.selectgo(0xc002160f70, 0xc002160dd0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002160d50 sp=0xc002160c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0021d44e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc002160fc0 sp=0xc002160d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc002160fe0 sp=0xc002160fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002160fe8 sp=0xc002160fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 873 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1007 [IO wait]: +runtime.gopark(0xc002330f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002330f40 sp=0xc002330f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002330f78 sp=0xc002330f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494ce8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002330f98 sp=0xc002330f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001ab0c20, 0xc0015dd8c0?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002330fe0 sp=0xc002330f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001ab0c00, {0xc0015dd8c0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0023310d0 sp=0xc002330fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc001ab0c00, {0xc0015dd8c0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002331130 sp=0xc0023310d0 pc=0x89342b +net.(*conn).Read(0xc000b79f88, {0xc0015dd8c0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0023311b8 sp=0xc002331130 pc=0x8b176d +net.(*UnixConn).Read(0xc000b79f88, {0xc0015dd8c0, 0x240, 0x240}) + :1 +0x54 fp=0xc002331200 sp=0xc0023311b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00304b620, {0xc0015dd8c0, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc002331260 sp=0xc002331200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00189fea8, {0x5b0f6c0, 0xc00304b620}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0023312c8 sp=0xc002331260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00189fc00, {0x5b0f840?, 0xc000b79f88}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc002331338 sp=0xc0023312c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00189fc00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc002331968 sp=0xc002331338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00189fc00, {0xc0023f6000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc002331a50 sp=0xc002331968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc002180740, {0xc0023f6000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc002331a98 sp=0xc002331a50 pc=0x10969ed +bufio.(*Reader).Read(0xc0021db500, {0xc0019392a0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002331b58 sp=0xc002331a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0021db500}, {0xc0019392a0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002331ba0 sp=0xc002331b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0019392a0, 0x9, 0x9}, {0x5b0f0a0, 0xc0021db500}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002331c40 sp=0xc002331ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001939260) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002331d78 sp=0xc002331c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0021d44e0, 0x616d656863732422?, 0x3a70747468223a22?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc002331ec8 sp=0xc002331d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc001ebfc20, {0x5b54e00?, 0xc0021d44e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc002331f80 sp=0xc002331ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc002331fe0 sp=0xc002331f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002331fe8 sp=0xc002331fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 873 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1018 [select]: +runtime.gopark(0xc002403fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002403f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002403de8 sp=0xc002403dc8 pc=0x4777ae +runtime.selectgo(0xc002403fb0, 0xc002403f70, 0xc0021565a0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002403f38 sp=0xc002403de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002403fe0 sp=0xc002403f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002403fe8 sp=0xc002403fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 992 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1019 [select]: +runtime.gopark(0xc002138fb0?, 0x2?, 0xb0?, 0x8d?, 0xc002138ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002138d58 sp=0xc002138d38 pc=0x4777ae +runtime.selectgo(0xc002138fb0, 0xc002138ee0, 0x8ab66c?, 0x0, 0x4b0571?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002138ea8 sp=0xc002138d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002138fe0 sp=0xc002138ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002138fe8 sp=0xc002138fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 992 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1020 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002404d60 sp=0xc002404d40 pc=0x4777ae +runtime.chanrecv(0xc0021575c0, 0xc002404e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002404dd8 sp=0xc002404d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002404e00 sp=0xc002404dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002404fe0 sp=0xc002404e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002404fe8 sp=0xc002404fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 992 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1102 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x14?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0018e1bd0 sp=0xc0018e1bb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0018e1c08 sp=0xc0018e1bd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494620, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0018e1c28 sp=0xc0018e1c08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc002a289a0, 0xc002a2a000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0018e1c70 sp=0xc0018e1c28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc002a28980, {0xc002a2a000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0018e1d60 sp=0xc0018e1c70 pc=0x50e5a5 +net.(*netFD).Read(0xc002a28980, {0xc002a2a000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0018e1dc0 sp=0xc0018e1d60 pc=0x89342b +net.(*conn).Read(0xc00007d6f0, {0xc002a2a000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0018e1e48 sp=0xc0018e1dc0 pc=0x8b176d +net.(*TCPConn).Read(0xc00007d6f0, {0xc002a2a000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0018e1e90 sp=0xc0018e1e48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc002a20440) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc0018e1f10 sp=0xc0018e1e90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc001659100) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc0018e1fc0 sp=0xc0018e1f10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc0018e1fe0 sp=0xc0018e1fc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0018e1fe8 sp=0xc0018e1fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 1017 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 1022 [select]: +runtime.gopark(0xc002406fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002406f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002406de8 sp=0xc002406dc8 pc=0x4777ae +runtime.selectgo(0xc002406fb0, 0xc002406f70, 0xc001be5920?, 0x0, 0xc001fe4540?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002406f38 sp=0xc002406de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002406fe0 sp=0xc002406f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002406fe8 sp=0xc002406fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 707 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1023 [select]: +runtime.gopark(0xc00234dfb0?, 0x2?, 0xb0?, 0xdd?, 0xc00234dee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00234dd58 sp=0xc00234dd38 pc=0x4777ae +runtime.selectgo(0xc00234dfb0, 0xc00234dee0, 0x1381849?, 0x0, 0xc000086ee8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00234dea8 sp=0xc00234dd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc00234dfe0 sp=0xc00234dea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00234dfe8 sp=0xc00234dfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1024 [chan receive]: +runtime.gopark(0xc000881b40?, 0xc002137eb8?, 0xbb?, 0xe2?, 0xc000881b20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002137d60 sp=0xc002137d40 pc=0x4777ae +runtime.chanrecv(0xc002157860, 0xc002137e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002137dd8 sp=0xc002137d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002137e00 sp=0xc002137dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002137fe0 sp=0xc002137e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002137fe8 sp=0xc002137fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 707 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1106 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x26?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e38bd0 sp=0xc001e38bb0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001e38c08 sp=0xc001e38bd0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494810, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001e38c28 sp=0xc001e38c08 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001f01820, 0xc00291e000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001e38c70 sp=0xc001e38c28 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001f01800, {0xc00291e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001e38d60 sp=0xc001e38c70 pc=0x50e5a5 +net.(*netFD).Read(0xc001f01800, {0xc00291e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001e38dc0 sp=0xc001e38d60 pc=0x89342b +net.(*conn).Read(0xc00007d730, {0xc00291e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001e38e48 sp=0xc001e38dc0 pc=0x8b176d +net.(*TCPConn).Read(0xc00007d730, {0xc00291e000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc001e38e90 sp=0xc001e38e48 pc=0x8d72f4 +github.com/nats-io/nats%2ego.(*natsReader).Read(0xc002853480) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1890 +0x1a9 fp=0xc001e38f10 sp=0xc001e38e90 pc=0x1f6d169 +github.com/nats-io/nats%2ego.(*Conn).readLoop(0xc00291a000) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2917 +0x1f8 fp=0xc001e38fc0 sp=0xc001e38f10 pc=0x1f75fb8 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x34 fp=0xc001e38fe0 sp=0xc001e38fc0 pc=0x1f71514 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e38fe8 sp=0xc001e38fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 886 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2261 +0x44e + +goroutine 1103 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0x28?, 0x91?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002347e18 sp=0xc002347df8 pc=0x4777ae +runtime.chanrecv(0xc002766de0, 0xc002347ed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002347e90 sp=0xc002347e18 pc=0x441325 +runtime.chanrecv2(0xc001659128?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc002347eb8 sp=0xc002347e90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc001659100) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc002347fc0 sp=0xc002347eb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc002347fe0 sp=0xc002347fc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002347fe8 sp=0xc002347fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 1017 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 1043 [select]: +runtime.gopark(0xc002337f70?, 0x2?, 0xb4?, 0x1b?, 0xc002337ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002337d20 sp=0xc002337d00 pc=0x4777ae +runtime.selectgo(0xc002337f70, 0xc002337eb8, 0x472ddb?, 0x0, 0x4ad421?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002337e70 sp=0xc002337d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001fabac0, {0x5b426e0, 0xc002338eb0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc002337fa0 sp=0xc002337e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc002337fe0 sp=0xc002337fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002337fe8 sp=0xc002337fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1017 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1044 [select]: +runtime.gopark(0xc002335f70?, 0x2?, 0xb4?, 0x1b?, 0xc002335ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002335d20 sp=0xc002335d00 pc=0x4777ae +runtime.selectgo(0xc002335f70, 0xc002335eb8, 0x4b3745?, 0x0, 0xc001fefb48?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002335e70 sp=0xc002335d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001fabb20, {0x5b426e0, 0xc002338f00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc002335fa0 sp=0xc002335e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc002335fe0 sp=0xc002335fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002335fe8 sp=0xc002335fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1017 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1037 [select]: +runtime.gopark(0xc0029e1f70?, 0x2?, 0xb4?, 0x1b?, 0xc0029e1ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029e1d20 sp=0xc0029e1d00 pc=0x4777ae +runtime.selectgo(0xc0029e1f70, 0xc0029e1eb8, 0xc00213deb8?, 0x0, 0xc00213df10?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029e1e70 sp=0xc0029e1d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002489380, {0x5b426e0, 0xc002055130}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0029e1fa0 sp=0xc0029e1e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0029e1fe0 sp=0xc0029e1fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029e1fe8 sp=0xc0029e1fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 823 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 879 [IO wait]: +runtime.gopark(0xc002358fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002358f98 sp=0xc002358f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002358fd0 sp=0xc002358f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494af8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002358ff0 sp=0xc002358fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc002305a20, 0xc00251ac00?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002359038 sp=0xc002358ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc002305a00, {0xc00251ac00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc002359128 sp=0xc002359038 pc=0x50e5a5 +net.(*netFD).Read(0xc002305a00, {0xc00251ac00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002359188 sp=0xc002359128 pc=0x89342b +net.(*conn).Read(0xc001d6fd18, {0xc00251ac00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc002359210 sp=0xc002359188 pc=0x8b176d +net.(*UnixConn).Read(0xc001d6fd18, {0xc00251ac00, 0x600, 0x600}) + :1 +0x54 fp=0xc002359258 sp=0xc002359210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0032a2a08, {0xc00251ac00, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0023592b8 sp=0xc002359258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0017cb0a8, {0x5b0f6c0, 0xc0032a2a08}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc002359320 sp=0xc0023592b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc0017cae00, {0x5b0f840?, 0xc001d6fd18}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc002359390 sp=0xc002359320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc0017cae00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0023599c0 sp=0xc002359390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc0017cae00, {0xc002526000, 0x8000, 0xc0030fc960?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc002359aa8 sp=0xc0023599c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d87a40, {0xc002526000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc002359af0 sp=0xc002359aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc00251daa0, {0xc0011d5b60, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002359bb0 sp=0xc002359af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00251daa0}, {0xc0011d5b60, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002359bf8 sp=0xc002359bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d5b60, 0x9, 0x9}, {0x5b0f0a0, 0xc00251daa0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002359c98 sp=0xc002359bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d5b20) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002359dd0 sp=0xc002359c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001df1200, 0xc001852b40?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc002359fb0 sp=0xc002359dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc002359fe0 sp=0xc002359fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002359fe8 sp=0xc002359fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1045 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1060 [select]: +runtime.gopark(0xc002470890?, 0x2?, 0x0?, 0x0?, 0xc00247083c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024706a8 sp=0xc002470688 pc=0x4777ae +runtime.selectgo(0xc002470890, 0xc002470838, 0x7f429a470b00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0024707f8 sp=0xc0024706a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0021e9d10, {0xc0024602e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0024708c0 sp=0xc0024707f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0021e9d10, {0xc0024602e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002470958 sp=0xc0024708c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00242fc50, {0xc0024602e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0024709a0 sp=0xc002470958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc00242fc50}, {0xc0024602e0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0024709e8 sp=0xc0024709a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c96000, {0xc0024602e0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002470a58 sp=0xc0024709e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0024602d0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002470b28 sp=0xc002470a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc00112bc00?, 0xc001c96000, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002470c18 sp=0xc002470b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0024602d0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc00255dc20}, 0xc002470d70?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002470cd0 sp=0xc002470c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001b9cc30, {0x43dcf60?, 0xc00255dc20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002470f08 sp=0xc002470cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002460420, {0x43dcf60, 0xc00255dc20}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002470ff0 sp=0xc002470f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002181440, {0x43dcf60, 0xc00255dc20}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002471028 sp=0xc002470ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc002447520) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc002471068 sp=0xc002471028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc002447520?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0024710a8 sp=0xc002471068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc002447530) + :1 +0x46 fp=0xc0024710e0 sp=0xc0024710a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc002447530}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc002471320 sp=0xc0024710e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc002447520}) + :1 +0xaf fp=0xc002471370 sp=0xc002471320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc002181440}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0024713b0 sp=0xc002471370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc002460420}, 0xc001e477d0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002471400 sp=0xc0024713b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc002460420}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002471490 sp=0xc002471400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9cc30}, 0xc001e477d0, 0xc002453200) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0024717d0 sp=0xc002471490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9cc30}, 0x3f252a0?, 0xc002447450?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002471848 sp=0xc0024717d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc001c96000, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002471db0 sp=0xc002471848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc001c96000, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002471f28 sp=0xc002471db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002471fe0 sp=0xc002471f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002471fe8 sp=0xc002471fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1061 [select]: +runtime.gopark(0xc001fba890?, 0x2?, 0x98?, 0xa7?, 0xc001fba83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001fba6a8 sp=0xc001fba688 pc=0x4777ae +runtime.selectgo(0xc001fba890, 0xc001fba838, 0x7f429a446500?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001fba7f8 sp=0xc001fba6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0021e9e00, {0xc00242ff90, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001fba8c0 sp=0xc001fba7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0021e9e00, {0xc00242ff90, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001fba958 sp=0xc001fba8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00242fda0, {0xc00242ff90, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001fba9a0 sp=0xc001fba958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc00242fda0}, {0xc00242ff90, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001fba9e8 sp=0xc001fba9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c96120, {0xc00242ff90, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001fbaa58 sp=0xc001fba9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc00242ff80, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001fbab28 sp=0xc001fbaa58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc002453080?, 0xc001c96120, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001fbac18 sp=0xc001fbab28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc00242ff80, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc002462230}, 0xc0010e4480?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001fbacd0 sp=0xc001fbac18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001b9cb40, {0x43dcf60?, 0xc002462230}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001fbaf08 sp=0xc001fbacd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0024600f0, {0x43dcf60, 0xc002462230}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001fbaff0 sp=0xc001fbaf08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002181360, {0x43dcf60, 0xc002462230}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001fbb028 sp=0xc001fbaff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc002447320) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001fbb068 sp=0xc001fbb028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc002447320?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001fbb0a8 sp=0xc001fbb068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc002447330) + :1 +0x46 fp=0xc001fbb0e0 sp=0xc001fbb0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0010e4630, {0x5b51368, 0xc002447330}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001fbb320 sp=0xc001fbb0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00158b250, {0x5b51038, 0xc002447320}) + :1 +0xaf fp=0xc001fbb370 sp=0xc001fbb320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00158b250}, {0x5b4a6f0?, 0xc002181360}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001fbb3b0 sp=0xc001fbb370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc0024600f0}, 0xc001e475d8?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001fbb400 sp=0xc001fbb3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00158b250}, {0x5b4ab30, 0xc0024600f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001fbb490 sp=0xc001fbb400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9cb40}, 0xc001e475d8, 0xc002452680) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001fbb7d0 sp=0xc001fbb490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00158b250}, {0x5b492f8, 0xc001b9cb40}, 0x3f252a0?, 0xc002447250?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001fbb848 sp=0xc001fbb7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc001c96120, 0xc0010e4810, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001fbbdb0 sp=0xc001fbb848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc001c96120, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001fbbf28 sp=0xc001fbbdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001fbbfe0 sp=0xc001fbbf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001fbbfe8 sp=0xc001fbbfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1351 [select]: +runtime.gopark(0xc0024cbf70?, 0x2?, 0xb4?, 0x1b?, 0xc0024cbebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024cbd20 sp=0xc0024cbd00 pc=0x4777ae +runtime.selectgo(0xc0024cbf70, 0xc0024cbeb8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0024cbe70 sp=0xc0024cbd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0021800e0, {0x5b426e0, 0xc001abb400}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0024cbfa0 sp=0xc0024cbe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0024cbfe0 sp=0xc0024cbfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024cbfe8 sp=0xc0024cbfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1320 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 881 [select]: +runtime.gopark(0xc003035e48?, 0x2?, 0x0?, 0x0?, 0xc003035dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc003035c30 sp=0xc003035c10 pc=0x4777ae +runtime.selectgo(0xc003035e48, 0xc003035dc8, 0xc003035dd8?, 0x0, 0xc002256700?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc003035d80 sp=0xc003035c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001abbef0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc003035e78 sp=0xc003035d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0025208c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc003035f30 sp=0xc003035e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc003035fe0 sp=0xc003035f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc003035fe8 sp=0xc003035fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 878 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1074 [select]: +runtime.gopark(0xc001d44f70?, 0x4?, 0xc0?, 0x2?, 0xc001d44dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d44c00 sp=0xc001d44be0 pc=0x4777ae +runtime.selectgo(0xc001d44f70, 0xc001d44dd0, 0x40a16a0?, 0x0, 0xc0018553f0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001d44d50 sp=0xc001d44c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002230340) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001d44fc0 sp=0xc001d44d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001d44fe0 sp=0xc001d44fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d44fe8 sp=0xc001d44fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 878 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1075 [IO wait]: +runtime.gopark(0xc002236f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002236f40 sp=0xc002236f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002236f78 sp=0xc002236f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494908, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002236f98 sp=0xc002236f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001ab10a0, 0xc0015ddd40?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002236fe0 sp=0xc002236f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001ab1080, {0xc0015ddd40, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0022370d0 sp=0xc002236fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc001ab1080, {0xc0015ddd40, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002237130 sp=0xc0022370d0 pc=0x89342b +net.(*conn).Read(0xc000b79fb8, {0xc0015ddd40, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0022371b8 sp=0xc002237130 pc=0x8b176d +net.(*UnixConn).Read(0xc000b79fb8, {0xc0015ddd40, 0x240, 0x240}) + :1 +0x54 fp=0xc002237200 sp=0xc0022371b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00304bb78, {0xc0015ddd40, 0x240, 0x240}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc002237260 sp=0xc002237200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc0022569a8, {0x5b0f6c0, 0xc00304bb78}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0022372c8 sp=0xc002237260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc002256700, {0x5b0f840?, 0xc000b79fb8}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc002237338 sp=0xc0022372c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc002256700, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc002237968 sp=0xc002237338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc002256700, {0xc002542000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc002237a50 sp=0xc002237968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d87ac0, {0xc002542000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc002237a98 sp=0xc002237a50 pc=0x10969ed +bufio.(*Reader).Read(0xc00251dda0, {0xc0011d5c40, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002237b58 sp=0xc002237a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00251dda0}, {0xc0011d5c40, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002237ba0 sp=0xc002237b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0011d5c40, 0x9, 0x9}, {0x5b0f0a0, 0xc00251dda0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002237c40 sp=0xc002237ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011d5c00) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002237d78 sp=0xc002237c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002230340, 0x78?, 0x1912368204258b60?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc002237ec8 sp=0xc002237d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc001852b40, {0x5b54e00?, 0xc002230340}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc002237f80 sp=0xc002237ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc002237fe0 sp=0xc002237f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002237fe8 sp=0xc002237fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 878 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1078 [select]: +runtime.gopark(0xc001d45fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001d45f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d45de8 sp=0xc001d45dc8 pc=0x4777ae +runtime.selectgo(0xc001d45fb0, 0xc001d45f70, 0xc00251de00?, 0x0, 0xc001d9dc80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001d45f38 sp=0xc001d45de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001d45fe0 sp=0xc001d45f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d45fe8 sp=0xc001d45fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1058 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1079 [select]: +runtime.gopark(0xc001d46fb0?, 0x2?, 0xb0?, 0x6d?, 0xc001d46ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d46d58 sp=0xc001d46d38 pc=0x4777ae +runtime.selectgo(0xc001d46fb0, 0xc001d46ee0, 0xe?, 0x0, 0x8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001d46ea8 sp=0xc001d46d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001d46fe0 sp=0xc001d46ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d46fe8 sp=0xc001d46fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1058 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1080 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002349d60 sp=0xc002349d40 pc=0x4777ae +runtime.chanrecv(0xc001d9df80, 0xc002349e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002349dd8 sp=0xc002349d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002349e00 sp=0xc002349dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002349fe0 sp=0xc002349e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002349fe8 sp=0xc002349fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1058 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1090 [chan receive]: +runtime.gopark(0x5b426a8?, 0x0?, 0xc8?, 0x7e?, 0x111478c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002401f08 sp=0xc002401ee8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002401f80 sp=0xc002401f08 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0xc001d6fd18?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002401fa8 sp=0xc002401f80 pc=0x440f72 +github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore.func3() + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:99 +0x4a fp=0xc002401fe0 sp=0xc002401fa8 pc=0x24a06aa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002401fe8 sp=0xc002401fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore in goroutine 823 + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:98 +0x4cb + +goroutine 1091 [select]: +runtime.gopark(0xc00235db10?, 0x13b0513?, 0x0?, 0x1?, 0xc002402ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002402d30 sp=0xc002402d10 pc=0x4777ae +runtime.selectgo(0xc002402f60, 0xc002402ec8, 0x4ad421?, 0x1, 0x12ad646?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002402e80 sp=0xc002402d30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc002341da0) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:64 +0x318 fp=0xc002402fc0 sp=0xc002402e80 pc=0x13b0538 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc002402fe0 sp=0xc002402fc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002402fe8 sp=0xc002402fe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 823 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 1092 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc002345930?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00235bdd8 sp=0xc00235bdb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002722990, 0xb) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc00235be28 sp=0xc00235bdd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc002722980) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc00235be58 sp=0xc00235be28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc001658a00, 0xc002345930) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc00235bfb0 sp=0xc00235be58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc00235bfe0 sp=0xc00235bfb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00235bfe8 sp=0xc00235bfe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 823 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 1104 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc00237d5a0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002346f08 sp=0xc002346ee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002a20410, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc002346f58 sp=0xc002346f08 pc=0x4a9339 +sync.(*Cond).Wait(0xc002a20400) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc002346f88 sp=0xc002346f58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc00237d5a0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc002346fc0 sp=0xc002346f88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc002346fe0 sp=0xc002346fc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002346fe8 sp=0xc002346fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 1017 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 1105 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc002a1b5f0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002355dd8 sp=0xc002355db8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002a20610, 0x1f) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc002355e28 sp=0xc002355dd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc002a20600) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc002355e58 sp=0xc002355e28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc001659100, 0xc002a1b5f0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc002355fb0 sp=0xc002355e58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc002355fe0 sp=0xc002355fb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002355fe8 sp=0xc002355fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 1017 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 1107 [chan receive]: +runtime.gopark(0xc00274ae40?, 0x4be4a5?, 0x28?, 0xa0?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00274ae18 sp=0xc00274adf8 pc=0x4777ae +runtime.chanrecv(0xc002767020, 0xc00274aed0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00274ae90 sp=0xc00274ae18 pc=0x441325 +runtime.chanrecv2(0xc00291a028?, 0x4aead9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc00274aeb8 sp=0xc00274ae90 pc=0x440f92 +github.com/nats-io/nats%2ego.(*Conn).flusher(0xc00291a000) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:3324 +0x1a5 fp=0xc00274afc0 sp=0xc00274aeb8 pc=0x1f792e5 +github.com/nats-io/nats%2ego.(*Conn).processConnectInit.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x34 fp=0xc00274afe0 sp=0xc00274afc0 pc=0x1f714b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00274afe8 sp=0xc00274afe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).processConnectInit in goroutine 886 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2262 +0x4cb + +goroutine 1108 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc0024890a0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002348f08 sp=0xc002348ee8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002853450, 0x0) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc002348f58 sp=0xc002348f08 pc=0x4a9339 +sync.(*Cond).Wait(0xc002853440) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc002348f88 sp=0xc002348f58 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher(0xc0024890a0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2842 +0x153 fp=0xc002348fc0 sp=0xc002348f88 pc=0x1f759b3 +github.com/nats-io/nats%2ego.Options.Connect.func2() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x34 fp=0xc002348fe0 sp=0xc002348fc0 pc=0x1f6a534 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002348fe8 sp=0xc002348fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.Options.Connect in goroutine 886 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:1502 +0x86b + +goroutine 1109 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002749f08 sp=0xc002749ee8 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002749f80 sp=0xc002749f08 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0xc001e2fa60?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002749fa8 sp=0xc002749f80 pc=0x440f72 +github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore.func3() + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:99 +0x4a fp=0xc002749fe0 sp=0xc002749fa8 pc=0x24a06aa +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002749fe8 sp=0xc002749fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.NewJetStreamStore in goroutine 886 + /home/ec2-user/opni/pkg/storage/jetstream/jetstream.go:98 +0x4cb + +goroutine 1110 [select]: +runtime.gopark(0xc002767270?, 0x13b0513?, 0x0?, 0x1?, 0xc00274decc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00274dd30 sp=0xc00274dd10 pc=0x4777ae +runtime.selectgo(0xc00274df60, 0xc00274dec8, 0xc00106e748?, 0x1, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00274de80 sp=0xc00274dd30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc002742c60) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:64 +0x318 fp=0xc00274dfc0 sp=0xc00274de80 pc=0x13b0538 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc00274dfe0 sp=0xc00274dfc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00274dfe8 sp=0xc00274dfe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 886 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 1111 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc002a1b930?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002357dd8 sp=0xc002357db8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002a20990, 0xa) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc002357e28 sp=0xc002357dd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc002a20980) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc002357e58 sp=0xc002357e28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc00291a000, 0xc002a1b930) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc002357fb0 sp=0xc002357e58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc002357fe0 sp=0xc002357fb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002357fe8 sp=0xc002357fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 886 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 1133 [chan receive]: +runtime.gopark(0x0?, 0xc002c01e10?, 0xa2?, 0x16?, 0xc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00212b318 sp=0xc00212b2f8 pc=0x4777ae +runtime.chanrecv(0xc0027e5c20, 0xc00212b478, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00212b390 sp=0xc00212b318 pc=0x441325 +runtime.chanrecv2(0xc000b6f040?, 0x5b426a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc00212b3b8 sp=0xc00212b390 pc=0x440f92 +github.com/rancher/opni/pkg/management.(*Server).WatchClusters(0xc0010c0e70, 0xc002c01e30, {0x5b4cb30, 0xc000b6f040}) + /home/ec2-user/opni/pkg/management/clusters.go:93 +0x26d fp=0xc00212b4a8 sp=0xc00212b3b8 pc=0x230a3cd +github.com/rancher/opni/pkg/apis/management/v1._Management_WatchClusters_Handler({0x4537040?, 0xc0010c0e70}, {0x5b4ab30, 0xc002c01e00}) + /home/ec2-user/opni/pkg/apis/management/v1/management_grpc.pb.go:736 +0xfa fp=0xc00212b508 sp=0xc00212b4a8 pc=0x178093a +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4537040, 0xc0010c0e70}, {0x5b492f8, 0xc000414960}, 0xc0028fce88, 0x47ec6c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc00212b848 sp=0xc00212b508 pc=0x12b0e02 +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001ebfc20, {0x5b54e00, 0xc0021d44e0}, 0xc0003bfd40, 0xc001be2420, 0x7716c60, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc00212bdb0 sp=0xc00212b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001ebfc20, {0x5b54e00, 0xc0021d44e0}, 0xc0003bfd40, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc00212bf28 sp=0xc00212bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00212bfe0 sp=0xc00212bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00212bfe8 sp=0xc00212bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1007 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1143 [select]: +runtime.gopark(0xc002189ed0?, 0x2?, 0xb4?, 0x1b?, 0xc002189e4c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002189c90 sp=0xc002189c70 pc=0x4777ae +runtime.selectgo(0xc002189ed0, 0xc002189e48, 0x0?, 0x0, 0x406e00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002189de0 sp=0xc002189c90 pc=0x48916b +github.com/rancher/opni/pkg/management.(*Server).listenAndServeGrpc(0xc0010c0e70, {0x5b426e0, 0xc0012c2730}) + /home/ec2-user/opni/pkg/management/server.go:215 +0x3cd fp=0xc002189f20 sp=0xc002189de0 pc=0x231e52d +github.com/rancher/opni/pkg/management.(*Server).ListenAndServe.func1() + /home/ec2-user/opni/pkg/management/server.go:181 +0x4b fp=0xc002189fa0 sp=0xc002189f20 pc=0x231e0ab +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc002189fe0 sp=0xc002189fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002189fe8 sp=0xc002189fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1084 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1233 [select]: +runtime.gopark(0xc002127f10?, 0x3?, 0xb4?, 0x1b?, 0xc002127e92?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002127ca8 sp=0xc002127c88 pc=0x4777ae +runtime.selectgo(0xc002127f10, 0xc002127e8c, 0x5b126c0?, 0x0, 0x1?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002127df8 sp=0xc002127ca8 pc=0x48916b +google.golang.org/grpc.(*addrConn).resetTransport(0xc001f13200) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:1368 +0x3ef fp=0xc002127f50 sp=0xc002127df8 pc=0x116334f +google.golang.org/grpc.(*addrConn).connect(0xc001f13200) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:993 +0x23b fp=0xc002127fc0 sp=0xc002127f50 pc=0x116033b +google.golang.org/grpc.(*acBalancerWrapper).Connect.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0x34 fp=0xc002127fe0 sp=0xc002127fc0 pc=0x1157434 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002127fe8 sp=0xc002127fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*acBalancerWrapper).Connect in goroutine 1040 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/balancer_conn_wrappers.go:398 +0xa7 + +goroutine 1135 [select]: +runtime.gopark(0xc002407fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002407f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002407de8 sp=0xc002407dc8 pc=0x4777ae +runtime.selectgo(0xc002407fb0, 0xc002407f70, 0x1000000?, 0x0, 0x116e9f5?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002407f38 sp=0xc002407de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002407fe0 sp=0xc002407f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002407fe8 sp=0xc002407fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1139 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1124 [chan receive]: +runtime.gopark(0x2106cfa?, 0x1177efa?, 0x67?, 0x6b?, 0xc001852b40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002059f00 sp=0xc002059ee0 pc=0x4777ae +runtime.chanrecv(0xc001bd48a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002059f78 sp=0xc002059f00 pc=0x441325 +runtime.chanrecv1(0xc001fee820?, 0xc001d4d6a0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002059fa0 sp=0xc002059f78 pc=0x440f72 +github.com/rancher/opni/plugins/metrics/pkg/cortex.NewClientSet.func1() + /home/ec2-user/opni/plugins/metrics/pkg/cortex/clientset.go:555 +0x53 fp=0xc002059fe0 sp=0xc002059fa0 pc=0x3d9cfb3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002059fe8 sp=0xc002059fe0 pc=0x4ad421 +created by github.com/rancher/opni/plugins/metrics/pkg/cortex.NewClientSet in goroutine 823 + /home/ec2-user/opni/plugins/metrics/pkg/cortex/clientset.go:554 +0x999 + +goroutine 1144 [select]: +runtime.gopark(0xc001755ed0?, 0x2?, 0xb4?, 0x1b?, 0xc001755e1c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001755c68 sp=0xc001755c48 pc=0x4777ae +runtime.selectgo(0xc001755ed0, 0xc001755e18, 0x1?, 0x0, 0xc0027d4600?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001755db8 sp=0xc001755c68 pc=0x48916b +github.com/rancher/opni/pkg/management.(*Server).listenAndServeHttp(0xc0010c0e70, {0x5b426e0?, 0xc0012c2730}) + /home/ec2-user/opni/pkg/management/server.go:248 +0x625 fp=0xc001755f20 sp=0xc001755db8 pc=0x231ed05 +github.com/rancher/opni/pkg/management.(*Server).ListenAndServe.func2() + /home/ec2-user/opni/pkg/management/server.go:189 +0x4b fp=0xc001755fa0 sp=0xc001755f20 pc=0x231dfab +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc001755fe0 sp=0xc001755fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001755fe8 sp=0xc001755fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1084 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1087 [select]: +runtime.gopark(0xc002008300?, 0x3?, 0xa0?, 0xea?, 0xc001575caa?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00246d7d0 sp=0xc00246d7b0 pc=0x4777ae +runtime.selectgo(0xc002008300, 0xc001575ca4, 0x0?, 0x0, 0xc001bf0240?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00246d920 sp=0xc00246d7d0 pc=0x48916b +reflect.rselect({0xc00246db08, 0x3, 0x452a85?}) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:589 +0x345 fp=0xc00246d9d0 sp=0xc00246d920 pc=0x4a8f05 +reflect.Select({0xc000b9a000, 0x3, 0x2?}) + /home/ec2-user/sdk/go1.21.0/src/reflect/value.go:3104 +0xb8a fp=0xc00246db98 sp=0xc00246d9d0 pc=0x57c44a +github.com/rancher/opni/pkg/util.WaitAll({0x5b426e0, 0xc00139cf00}, 0xc001bf0200, {0xc00246de08, 0x2, 0x2}) + /home/ec2-user/opni/pkg/util/servers.go:51 +0x3ea fp=0xc00246dd18 sp=0xc00246db98 pc=0x176ae8a +github.com/rancher/opni/pkg/gateway.(*GatewayHTTPServer).ListenAndServe(0xc0013042a0, {0x5b426e0, 0xc0007639a0}) + /home/ec2-user/opni/pkg/gateway/http.go:185 +0x5a7 fp=0xc00246de68 sp=0xc00246dd18 pc=0x22b8427 +github.com/rancher/opni/pkg/gateway.(*Gateway).ListenAndServe.func1() + /home/ec2-user/opni/pkg/gateway/gateway.go:327 +0x86 fp=0xc00246dfa0 sp=0xc00246de68 pc=0x22b28a6 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc00246dfe0 sp=0xc00246dfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00246dfe8 sp=0xc00246dfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1085 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1088 [select]: +runtime.gopark(0xc001bebe38?, 0x2?, 0xb4?, 0x1b?, 0xc001bebc5c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001beba58 sp=0xc001beba38 pc=0x4777ae +runtime.selectgo(0xc001bebe38, 0xc001bebc58, 0x0?, 0x0, 0x4113100?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bebba8 sp=0xc001beba58 pc=0x48916b +github.com/rancher/opni/pkg/gateway.(*GatewayGRPCServer).ListenAndServe(0xc001df9b00, {0x5b426e0, 0xc0007639a0}) + /home/ec2-user/opni/pkg/gateway/grpc.go:107 +0x106f fp=0xc001bebe68 sp=0xc001bebba8 pc=0x22b4b0f +github.com/rancher/opni/pkg/gateway.(*Gateway).ListenAndServe.func2() + /home/ec2-user/opni/pkg/gateway/gateway.go:338 +0x86 fp=0xc001bebfa0 sp=0xc001bebe68 pc=0x22b2686 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc001bebfe0 sp=0xc001bebfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bebfe8 sp=0xc001bebfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1085 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1173 [chan receive]: +runtime.gopark(0xc001441b00?, 0x0?, 0x0?, 0x4e?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002755ee8 sp=0xc002755ec8 pc=0x4777ae +runtime.chanrecv(0xc0021fe9c0, 0xc002755fa0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002755f60 sp=0xc002755ee8 pc=0x441325 +runtime.chanrecv2(0x0?, 0xc000b31560?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc002755f88 sp=0xc002755f60 pc=0x440f92 +google.golang.org/grpc.(*Server).serverWorker(0xc00170e000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:593 +0x85 fp=0xc002755fc0 sp=0xc002755f88 pc=0x11744a5 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc002755fe0 sp=0xc002755fc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002755fe8 sp=0xc002755fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 1088 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 1175 [chan receive]: +runtime.gopark(0xc0018f5e60?, 0x0?, 0x0?, 0x4e?, 0xc001000000?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027ddee8 sp=0xc0027ddec8 pc=0x4777ae +runtime.chanrecv(0xc0021fe9c0, 0xc0027ddfa0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0027ddf60 sp=0xc0027ddee8 pc=0x441325 +runtime.chanrecv2(0x1?, 0xc002488880?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0027ddf88 sp=0xc0027ddf60 pc=0x440f92 +google.golang.org/grpc.(*Server).serverWorker(0xc00170e000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:593 +0x85 fp=0xc0027ddfc0 sp=0xc0027ddf88 pc=0x11744a5 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc0027ddfe0 sp=0xc0027ddfc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0027ddfe8 sp=0xc0027ddfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 1088 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 1145 [select]: +runtime.gopark(0xc0017c9f70?, 0x2?, 0xb4?, 0x1b?, 0xc0017c9ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017c9d20 sp=0xc0017c9d00 pc=0x4777ae +runtime.selectgo(0xc0017c9f70, 0xc0017c9eb8, 0x4b3745?, 0x0, 0xc00106e748?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017c9e70 sp=0xc0017c9d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0027849a0, {0x5b426e0, 0xc0012c29b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0017c9fa0 sp=0xc0017c9e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0017c9fe0 sp=0xc0017c9fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017c9fe8 sp=0xc0017c9fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1144 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1146 [select]: +runtime.gopark(0xc0029d9f70?, 0x2?, 0xb4?, 0x1b?, 0xc0029d9ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029d9d20 sp=0xc0029d9d00 pc=0x4777ae +runtime.selectgo(0xc0029d9f70, 0xc0029d9eb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029d9e70 sp=0xc0029d9d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002784a00, {0x5b426e0, 0xc0012c2a00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0029d9fa0 sp=0xc0029d9e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0029d9fe0 sp=0xc0029d9fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029d9fe8 sp=0xc0029d9fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1144 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1149 [select]: +runtime.gopark(0xc0017c3f70?, 0x2?, 0xb4?, 0x1b?, 0xc0017c3ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017c3d20 sp=0xc0017c3d00 pc=0x4777ae +runtime.selectgo(0xc0017c3f70, 0xc0017c3eb8, 0xc001b70600?, 0x0, 0x440100?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017c3e70 sp=0xc0017c3d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0027854c0, {0x5b426e0, 0xc000b68870}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0017c3fa0 sp=0xc0017c3e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0017c3fe0 sp=0xc0017c3fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017c3fe8 sp=0xc0017c3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 609 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1176 [chan receive]: +runtime.gopark(0xc001d8a7e0?, 0x0?, 0x0?, 0x4e?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027e1ee8 sp=0xc0027e1ec8 pc=0x4777ae +runtime.chanrecv(0xc0021fe9c0, 0xc0027e1fa0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0027e1f60 sp=0xc0027e1ee8 pc=0x441325 +runtime.chanrecv2(0xc0000fd360?, 0xc002bf88e0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0027e1f88 sp=0xc0027e1f60 pc=0x440f92 +google.golang.org/grpc.(*Server).serverWorker(0xc00170e000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:593 +0x85 fp=0xc0027e1fc0 sp=0xc0027e1f88 pc=0x11744a5 +google.golang.org/grpc.(*Server).initServerWorkers.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0x34 fp=0xc0027e1fe0 sp=0xc0027e1fc0 pc=0x1174994 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0027e1fe8 sp=0xc0027e1fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).initServerWorkers in goroutine 1088 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:612 +0xa5 + +goroutine 1160 [select]: +runtime.gopark(0xc001980ed8?, 0x2?, 0xb4?, 0x1b?, 0xc001980e84?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001980cf8 sp=0xc001980cd8 pc=0x4777ae +runtime.selectgo(0xc001980ed8, 0xc001980e80, 0xc001980ea8?, 0x0, 0xc001980e90?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001980e48 sp=0xc001980cf8 pc=0x48916b +github.com/rancher/opni/pkg/util.ServeHandler({0x5b426e0, 0xc00139cf00}, {0x5b13c00?, 0xc0012c4b60}, {0x5b2a6d0?, 0xc00296ff00}) + /home/ec2-user/opni/pkg/util/servers.go:21 +0x26a fp=0xc001980f08 sp=0xc001980e48 pc=0x176a94a +github.com/rancher/opni/pkg/gateway.(*GatewayHTTPServer).ListenAndServe.func2() + /home/ec2-user/opni/pkg/gateway/http.go:182 +0x1b9 fp=0xc001980fa0 sp=0xc001980f08 pc=0x22b8699 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc001980fe0 sp=0xc001980fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001980fe8 sp=0xc001980fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1087 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1177 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029359e0 sp=0xc0029359c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002935a18 sp=0xc0029359e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163adc0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002935a38 sp=0xc002935a18 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0016f8a20, 0x4b0501?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002935a80 sp=0xc002935a38 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0016f8a00) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc002935b70 sp=0xc002935a80 pc=0x513f65 +net.(*netFD).accept(0xc0016f8a00) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc002935c68 sp=0xc002935b70 pc=0x8967be +net.(*TCPListener).accept(0xc0016353c0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc002935cb0 sp=0xc002935c68 pc=0x8c383e +net.(*TCPListener).Accept(0xc0016353c0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc002935d20 sp=0xc002935cb0 pc=0x8c1ba5 +google.golang.org/grpc.(*Server).Serve(0xc00170e000, {0x5b2a6d0, 0xc0016353c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc002935f50 sp=0xc002935d20 pc=0x117789e +github.com/rancher/opni/pkg/gateway.(*GatewayGRPCServer).ListenAndServe.func1() + /home/ec2-user/opni/pkg/gateway/grpc.go:105 +0x47 fp=0xc002935fa0 sp=0xc002935f50 pc=0x22b4c27 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc002935fe0 sp=0xc002935fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002935fe8 sp=0xc002935fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1088 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1161 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00215c9e0 sp=0xc00215c9c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00215ca18 sp=0xc00215c9e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163a6f8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00215ca38 sp=0xc00215ca18 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182c120, 0x4b0501?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00215ca80 sp=0xc00215ca38 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00182c100) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc00215cb70 sp=0xc00215ca80 pc=0x513f65 +net.(*netFD).accept(0xc00182c100) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc00215cc68 sp=0xc00215cb70 pc=0x8967be +net.(*TCPListener).accept(0xc00296ff00) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc00215ccb0 sp=0xc00215cc68 pc=0x8c383e +net.(*TCPListener).Accept(0xc00296ff00) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc00215cd20 sp=0xc00215ccb0 pc=0x8c1ba5 +net/http.(*onceCloseListener).Accept(0xc002008330) + :1 +0x44 fp=0xc00215cd58 sp=0xc00215cd20 pc=0xab5544 +net/http.(*Server).Serve(0xc000b9c000, {0x5b2a6d0, 0xc00296ff00}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc00215cf50 sp=0xc00215cd58 pc=0xa7859f +github.com/rancher/opni/pkg/util.ServeHandler.func1() + /home/ec2-user/opni/pkg/util/servers.go:19 +0x47 fp=0xc00215cfa0 sp=0xc00215cf50 pc=0x176aa67 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc00215cfe0 sp=0xc00215cfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00215cfe8 sp=0xc00215cfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1160 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1178 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001823988 sp=0xc001823968 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0018239c0 sp=0xc001823988 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424163abd0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0018239e0 sp=0xc0018239c0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182c0a0, 0xc001738301?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001823a28 sp=0xc0018239e0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00182c080) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc001823b18 sp=0xc001823a28 pc=0x513f65 +net.(*netFD).accept(0xc00182c080) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc001823c10 sp=0xc001823b18 pc=0x8967be +net.(*TCPListener).accept(0xc00296fea0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc001823c58 sp=0xc001823c10 pc=0x8c383e +net.(*TCPListener).Accept(0xc00296fea0) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc001823cc8 sp=0xc001823c58 pc=0x8c1ba5 +crypto/tls.(*listener).Accept(0xc0028fd4d0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/tls.go:66 +0x43 fp=0xc001823d20 sp=0xc001823cc8 pc=0x9b2d43 +net/http.(*onceCloseListener).Accept(0xc002172ed0) + :1 +0x44 fp=0xc001823d58 sp=0xc001823d20 pc=0xab5544 +net/http.(*Server).Serve(0xc0016702d0, {0x5b2a5b0, 0xc0028fd4d0}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc001823f50 sp=0xc001823d58 pc=0xa7859f +github.com/rancher/opni/pkg/util.ServeHandler.func1() + /home/ec2-user/opni/pkg/util/servers.go:19 +0x47 fp=0xc001823fa0 sp=0xc001823f50 pc=0x176aa67 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc001823fe0 sp=0xc001823fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001823fe8 sp=0xc001823fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1159 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1317 [syscall]: +syscall.Syscall6(0x100000045?, 0x47efdc8?, 0xc0021fceb0?, 0xc0020099e0?, 0xc00099bdc8?, 0xc00099bdd8?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/syscall/syscall_linux.go:91 +0x3d fp=0xc002748d38 sp=0xc002748cb0 pc=0x4d1b1d +os.(*Process).blockUntilWaitable(0xc0009537d0) + /home/ec2-user/sdk/go1.21.0/src/os/wait_waitid.go:32 +0x9f fp=0xc002748e18 sp=0xc002748d38 pc=0x529c5f +os.(*Process).wait(0xc0009537d0) + /home/ec2-user/sdk/go1.21.0/src/os/exec_unix.go:22 +0x4b fp=0xc002748e98 sp=0xc002748e18 pc=0x51e0ab +os.(*Process).Wait(...) + /home/ec2-user/sdk/go1.21.0/src/os/exec.go:134 +os/exec.(*Cmd).Wait(0xc0026c3340) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:890 +0xb9 fp=0xc002748f60 sp=0xc002748e98 pc=0x657279 +github.com/onsi/gomega/gexec.(*Session).monitorForExit(0xc002460840, 0x0?) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:196 +0x3f fp=0xc002748fb0 sp=0xc002748f60 pc=0x233731f +github.com/onsi/gomega/gexec.Start.func1() + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x45 fp=0xc002748fe0 sp=0xc002748fb0 pc=0x2336d65 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002748fe8 sp=0xc002748fe0 pc=0x4ad421 +created by github.com/onsi/gomega/gexec.Start in goroutine 1250 + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x9eb + +goroutine 1163 [select]: +runtime.gopark(0xc00190be48?, 0x2?, 0x0?, 0x0?, 0xc00190bdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00190bc30 sp=0xc00190bc10 pc=0x4777ae +runtime.selectgo(0xc00190be48, 0xc00190bdc8, 0xc00190bdd8?, 0x0, 0xc000641580?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00190bd80 sp=0xc00190bc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139d270, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00190be78 sp=0xc00190bd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0018b8770) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00190bf30 sp=0xc00190be78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc00190bfe0 sp=0xc00190bf30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00190bfe8 sp=0xc00190bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1162 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1164 [select]: +runtime.gopark(0xc00181ff70?, 0x4?, 0x58?, 0xfc?, 0xc00181fdd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00181fc00 sp=0xc00181fbe0 pc=0x4777ae +runtime.selectgo(0xc00181ff70, 0xc00181fdd0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00181fd50 sp=0xc00181fc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0021d5520) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc00181ffc0 sp=0xc00181fd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc00181ffe0 sp=0xc00181ffc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00181ffe8 sp=0xc00181ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1162 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1165 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00215d7d8 sp=0xc00215d7b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00215d810 sp=0xc00215d7d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494430, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00215d830 sp=0xc00215d810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182c1a0, 0xc001880000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00215d878 sp=0xc00215d830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00182c180, {0xc001880000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00215d968 sp=0xc00215d878 pc=0x50e5a5 +net.(*netFD).Read(0xc00182c180, {0xc001880000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00215d9c8 sp=0xc00215d968 pc=0x89342b +net.(*conn).Read(0xc000641580, {0xc001880000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00215da50 sp=0xc00215d9c8 pc=0x8b176d +net.(*TCPConn).Read(0xc000641580, {0xc001880000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc00215da98 sp=0xc00215da50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc000c4e540, {0xc000b9a120, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00215db58 sp=0xc00215da98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000c4e540}, {0xc000b9a120, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00215dba0 sp=0xc00215db58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000b9a120, 0x9, 0x9}, {0x5b0f0a0, 0xc000c4e540}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00215dc40 sp=0xc00215dba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000b9a0e0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00215dd78 sp=0xc00215dc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0021d5520, 0x28?, 0x5b426a8?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc00215dec8 sp=0xc00215dd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c1e0, {0x5b54e00?, 0xc0021d5520}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc00215df80 sp=0xc00215dec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc00215dfe0 sp=0xc00215df80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00215dfe8 sp=0xc00215dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1162 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1212 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0022db830 sp=0xc0022db810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0022db868 sp=0xc0022db830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243493f58, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0022db888 sp=0xc0022db868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0016f8d20, 0xc0022c0000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0022db8d0 sp=0xc0022db888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0016f8d00, {0xc0022c0000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0022db9c0 sp=0xc0022db8d0 pc=0x50e5a5 +net.(*netFD).Read(0xc0016f8d00, {0xc0022c0000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0022dba20 sp=0xc0022db9c0 pc=0x89342b +net.(*conn).Read(0xc001d6e9f8, {0xc0022c0000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0022dbaa8 sp=0xc0022dba20 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6e9f8, {0xc0022c0000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0022dbaf0 sp=0xc0022dbaa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc00150d9e0, {0xc0017031c0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0022dbbb0 sp=0xc0022dbaf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00150d9e0}, {0xc0017031c0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0022dbbf8 sp=0xc0022dbbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0017031c0, 0x9, 0x9}, {0x5b0f0a0, 0xc00150d9e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0022dbc98 sp=0xc0022dbbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001703180) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0022dbdd0 sp=0xc0022dbc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000eedb00, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0022dbfb0 sp=0xc0022dbdd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0022dbfe0 sp=0xc0022dbfb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0022dbfe8 sp=0xc0022dbfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1151 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1183 [IO wait]: +runtime.gopark(0x448485?, 0x431af0?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00293b708 sp=0xc00293b6e8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00293b740 sp=0xc00293b708 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494338, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00293b760 sp=0xc00293b740 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0016f8b20, 0xc001847000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00293b7a8 sp=0xc00293b760 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0016f8b00, {0xc001847000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00293b898 sp=0xc00293b7a8 pc=0x50e5a5 +net.(*netFD).Read(0xc0016f8b00, {0xc001847000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00293b8f8 sp=0xc00293b898 pc=0x89342b +net.(*conn).Read(0xc001d6e968, {0xc001847000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00293b980 sp=0xc00293b8f8 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6e968, {0xc001847000, 0x1000, 0x1000}) + :1 +0x54 fp=0xc00293b9c8 sp=0xc00293b980 pc=0x8d72f4 +net/http.(*persistConn).Read(0xc0012c17a0, {0xc001847000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1954 +0xf9 fp=0xc00293ba48 sp=0xc00293b9c8 pc=0xa9c1d9 +bufio.(*Reader).fill(0xc00150d5c0) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:113 +0x29a fp=0xc00293bad8 sp=0xc00293ba48 pc=0x5f863a +bufio.(*Reader).Peek(0xc00150d5c0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:151 +0xc7 fp=0xc00293bb60 sp=0xc00293bad8 pc=0x5f88e7 +net/http.(*persistConn).readLoop(0xc0012c17a0) + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:2118 +0x2db fp=0xc00293bfc0 sp=0xc00293bb60 pc=0xa9d41b +net/http.(*Transport).dialConn.func5() + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1776 +0x34 fp=0xc00293bfe0 sp=0xc00293bfc0 pc=0xa9aef4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00293bfe8 sp=0xc00293bfe0 pc=0x4ad421 +created by net/http.(*Transport).dialConn in goroutine 1179 + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1776 +0x24da + +goroutine 1184 [select]: +runtime.gopark(0xc002746f90?, 0x2?, 0x10?, 0x6d?, 0xc002746e6c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002746cc8 sp=0xc002746ca8 pc=0x4777ae +runtime.selectgo(0xc002746f90, 0xc002746e68, 0xc0011cae80?, 0x0, 0xc0021732c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002746e18 sp=0xc002746cc8 pc=0x48916b +net/http.(*persistConn).writeLoop(0xc0012c17a0) + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:2421 +0x196 fp=0xc002746fc0 sp=0xc002746e18 pc=0xa9fe36 +net/http.(*Transport).dialConn.func6() + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1777 +0x34 fp=0xc002746fe0 sp=0xc002746fc0 pc=0xa9ae94 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002746fe8 sp=0xc002746fe0 pc=0x4ad421 +created by net/http.(*Transport).dialConn in goroutine 1179 + /home/ec2-user/sdk/go1.21.0/src/net/http/transport.go:1777 +0x256b + +goroutine 1206 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017c5830 sp=0xc0017c5810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0017c5868 sp=0xc0017c5830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494148, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0017c5888 sp=0xc0017c5868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0016f8ba0, 0xc0021de000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0017c58d0 sp=0xc0017c5888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0016f8b80, {0xc0021de000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0017c59c0 sp=0xc0017c58d0 pc=0x50e5a5 +net.(*netFD).Read(0xc0016f8b80, {0xc0021de000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0017c5a20 sp=0xc0017c59c0 pc=0x89342b +net.(*conn).Read(0xc001d6e988, {0xc0021de000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0017c5aa8 sp=0xc0017c5a20 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6e988, {0xc0021de000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0017c5af0 sp=0xc0017c5aa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc00150d740, {0xc001703000, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0017c5bb0 sp=0xc0017c5af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00150d740}, {0xc001703000, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0017c5bf8 sp=0xc0017c5bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001703000, 0x9, 0x9}, {0x5b0f0a0, 0xc00150d740}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0017c5c98 sp=0xc0017c5bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001702fc0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0017c5dd0 sp=0xc0017c5c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000eed680, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0017c5fb0 sp=0xc0017c5dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0017c5fe0 sp=0xc0017c5fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017c5fe8 sp=0xc0017c5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1185 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1150 [select]: +runtime.gopark(0xc001f3ff70?, 0x2?, 0xb4?, 0x1b?, 0xc001f3febc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f3fd20 sp=0xc001f3fd00 pc=0x4777ae +runtime.selectgo(0xc001f3ff70, 0xc001f3feb8, 0x4b3745?, 0x0, 0xc00106e748?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f3fe70 sp=0xc001f3fd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002785520, {0x5b426e0, 0xc000b688c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001f3ffa0 sp=0xc001f3fe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001f3ffe0 sp=0xc001f3ffa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f3ffe8 sp=0xc001f3ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 609 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1207 [select]: +runtime.gopark(0xc0021f3e70?, 0x2?, 0x0?, 0x0?, 0xc0021f3df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0021f3c58 sp=0xc0021f3c38 pc=0x4777ae +runtime.selectgo(0xc0021f3e70, 0xc0021f3df0, 0xc0021f3e00?, 0x0, 0xc001d6e988?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0021f3da8 sp=0xc0021f3c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00099bdb0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0021f3ea0 sp=0xc0021f3da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001704c40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0021f3f58 sp=0xc0021f3ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0021f3fe0 sp=0xc0021f3f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0021f3fe8 sp=0xc0021f3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1185 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1214 [select]: +runtime.gopark(0xc00232fe48?, 0x2?, 0x0?, 0x0?, 0xc00232fdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00232fc30 sp=0xc00232fc10 pc=0x4777ae +runtime.selectgo(0xc00232fe48, 0xc00232fdc8, 0xc00232fdd8?, 0x0, 0xc001d6e9e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00232fd80 sp=0xc00232fc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001a0c410, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00232fe78 sp=0xc00232fd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001704d90) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00232ff30 sp=0xc00232fe78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc00232ffe0 sp=0xc00232ff30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00232ffe8 sp=0xc00232ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1210 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1168 [select]: +runtime.gopark(0xc00226fe48?, 0x2?, 0x0?, 0x0?, 0xc00226fdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00226fc30 sp=0xc00226fc10 pc=0x4777ae +runtime.selectgo(0xc00226fe48, 0xc00226fdc8, 0xc0018b87e0?, 0x0, 0xc0006415a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00226fd80 sp=0xc00226fc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00139d950, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc00226fe78 sp=0xc00226fd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0018b87e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc00226ff30 sp=0xc00226fe78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc00226ffe0 sp=0xc00226ff30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00226ffe8 sp=0xc00226ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1167 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1169 [select]: +runtime.gopark(0xc001821f70?, 0x4?, 0xd8?, 0xd7?, 0xc001821dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001821c00 sp=0xc001821be0 pc=0x4777ae +runtime.selectgo(0xc001821f70, 0xc001821dd0, 0x40a16a0?, 0x0, 0xc001bf03f0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001821d50 sp=0xc001821c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000b9e000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001821fc0 sp=0xc001821d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001821fe0 sp=0xc001821fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001821fe8 sp=0xc001821fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1167 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1218 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017337d8 sp=0xc0017337b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001733810 sp=0xc0017337d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f4243494050, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001733830 sp=0xc001733810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182c2a0, 0xc002168000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001733878 sp=0xc001733830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00182c280, {0xc002168000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001733968 sp=0xc001733878 pc=0x50e5a5 +net.(*netFD).Read(0xc00182c280, {0xc002168000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0017339c8 sp=0xc001733968 pc=0x89342b +net.(*conn).Read(0xc0006415a0, {0xc002168000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001733a50 sp=0xc0017339c8 pc=0x8b176d +net.(*TCPConn).Read(0xc0006415a0, {0xc002168000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc001733a98 sp=0xc001733a50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc0010bc300, {0xc000b9a2e0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001733b58 sp=0xc001733a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc0010bc300}, {0xc000b9a2e0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001733ba0 sp=0xc001733b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc000b9a2e0, 0x9, 0x9}, {0x5b0f0a0, 0xc0010bc300}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001733c40 sp=0xc001733ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc000b9a2a0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001733d78 sp=0xc001733c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000b9e000, 0x28?, 0x5b426a8?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001733ec8 sp=0xc001733d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c1e0, {0x5b54e00?, 0xc000b9e000}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001733f80 sp=0xc001733ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001733fe0 sp=0xc001733f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001733fe8 sp=0xc001733fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1167 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1215 [select]: +runtime.gopark(0xc00181df70?, 0x4?, 0x0?, 0x0?, 0xc00181ddd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00181dc00 sp=0xc00181dbe0 pc=0x4777ae +runtime.selectgo(0xc00181df70, 0xc00181ddd0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00181dd50 sp=0xc00181dc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001756680) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc00181dfc0 sp=0xc00181dd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc00181dfe0 sp=0xc00181dfc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00181dfe8 sp=0xc00181dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1210 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1153 [select]: +runtime.gopark(0xc001f3df70?, 0x2?, 0xb4?, 0x1b?, 0xc001f3debc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f3dd20 sp=0xc001f3dd00 pc=0x4777ae +runtime.selectgo(0xc001f3df70, 0xc001f3deb8, 0x4aead9?, 0x0, 0x4b0571?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f3de70 sp=0xc001f3dd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002785880, {0x5b426e0, 0xc0007c0690}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001f3dfa0 sp=0xc001f3de70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001f3dfe0 sp=0xc001f3dfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f3dfe8 sp=0xc001f3dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 609 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1234 [select]: +runtime.gopark(0xc001f3bf70?, 0x2?, 0xb4?, 0x1b?, 0xc001f3bebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f3bd20 sp=0xc001f3bd00 pc=0x4777ae +runtime.selectgo(0xc001f3bf70, 0xc001f3beb8, 0x4aead9?, 0x0, 0x4b05a9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f3be70 sp=0xc001f3bd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0027858e0, {0x5b426e0, 0xc0007c06e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc001f3bfa0 sp=0xc001f3be70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc001f3bfe0 sp=0xc001f3bfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f3bfe8 sp=0xc001f3bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 609 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1356 [select]: +runtime.gopark(0xc001f39da8?, 0x2?, 0x60?, 0x98?, 0xc001f39a94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f39818 sp=0xc001f397f8 pc=0x4777ae +runtime.selectgo(0xc001f39da8, 0xc001f39a90, 0x0?, 0x0, 0x1860a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001f39968 sp=0xc001f39818 pc=0x48916b +github.com/hashicorp/go-plugin.Serve(0xc001e54ea0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:474 +0x2091 fp=0xc001f39fc0 sp=0xc001f39968 pc=0x21665b1 +github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins.func4() + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x34 fp=0xc001f39fe0 sp=0xc001f39fc0 pc=0x24cd234 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f39fe8 sp=0xc001f39fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins in goroutine 1320 + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x76b + +goroutine 1213 [select]: +runtime.gopark(0xc0022e7e70?, 0x2?, 0x0?, 0x0?, 0xc0022e7df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0022e7c58 sp=0xc0022e7c38 pc=0x4777ae +runtime.selectgo(0xc0022e7e70, 0xc0022e7df0, 0xc0022e7e00?, 0x0, 0xc001d6e9f8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0022e7da8 sp=0xc0022e7c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001a0c6e0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0022e7ea0 sp=0xc0022e7da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001704d20) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0022e7f58 sp=0xc0022e7ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0022e7fe0 sp=0xc0022e7f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0022e7fe8 sp=0xc0022e7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1151 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1268 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002747830 sp=0xc002747810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002747868 sp=0xc002747830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d960, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002747888 sp=0xc002747868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0027d59a0, 0xc002472000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0027478d0 sp=0xc002747888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0027d5980, {0xc002472000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0027479c0 sp=0xc0027478d0 pc=0x50e5a5 +net.(*netFD).Read(0xc0027d5980, {0xc002472000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002747a20 sp=0xc0027479c0 pc=0x89342b +net.(*conn).Read(0xc000bdffd0, {0xc002472000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc002747aa8 sp=0xc002747a20 pc=0x8b176d +net.(*TCPConn).Read(0xc000bdffd0, {0xc002472000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc002747af0 sp=0xc002747aa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc000d65440, {0xc001ebde00, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002747bb0 sp=0xc002747af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000d65440}, {0xc001ebde00, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002747bf8 sp=0xc002747bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001ebde00, 0x9, 0x9}, {0x5b0f0a0, 0xc000d65440}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002747c98 sp=0xc002747bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001ebddc0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002747dd0 sp=0xc002747c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00165a900, 0x0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc002747fb0 sp=0xc002747dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc002747fe0 sp=0xc002747fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002747fe8 sp=0xc002747fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1247 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1269 [select]: +runtime.gopark(0xc002491e70?, 0x2?, 0x0?, 0x0?, 0xc002491df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002491c58 sp=0xc002491c38 pc=0x4777ae +runtime.selectgo(0xc002491e70, 0xc002491df0, 0xc002491e00?, 0x0, 0xc000bdffd0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002491da8 sp=0xc002491c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0007c1d10, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002491ea0 sp=0xc002491da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0023bcaf0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002491f58 sp=0xc002491ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc002491fe0 sp=0xc002491f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002491fe8 sp=0xc002491fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1247 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1216 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00172f7d8 sp=0xc00172f7b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00172f810 sp=0xc00172f7d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169df30, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00172f830 sp=0xc00172f810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0016f8da0, 0xc0022a8000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00172f878 sp=0xc00172f830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0016f8d80, {0xc0022a8000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00172f968 sp=0xc00172f878 pc=0x50e5a5 +net.(*netFD).Read(0xc0016f8d80, {0xc0022a8000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00172f9c8 sp=0xc00172f968 pc=0x89342b +net.(*conn).Read(0xc001d6e9e0, {0xc0022a8000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00172fa50 sp=0xc00172f9c8 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6e9e0, {0xc0022a8000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc00172fa98 sp=0xc00172fa50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc00150d980, {0xc0017030e0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00172fb58 sp=0xc00172fa98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00150d980}, {0xc0017030e0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00172fba0 sp=0xc00172fb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0017030e0, 0x9, 0x9}, {0x5b0f0a0, 0xc00150d980}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00172fc40 sp=0xc00172fba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0017030a0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00172fd78 sp=0xc00172fc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001756680, 0x28?, 0x4258b60?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc00172fec8 sp=0xc00172fd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c1e0, {0x5b54e00?, 0xc001756680}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc00172ff80 sp=0xc00172fec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc00172ffe0 sp=0xc00172ff80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00172ffe8 sp=0xc00172ffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1210 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1242 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0022df830 sp=0xc0022df810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0022df868 sp=0xc0022df830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169de38, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0022df888 sp=0xc0022df868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0027d57a0, 0xc002362000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0022df8d0 sp=0xc0022df888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0027d5780, {0xc002362000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0022df9c0 sp=0xc0022df8d0 pc=0x50e5a5 +net.(*netFD).Read(0xc0027d5780, {0xc002362000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0022dfa20 sp=0xc0022df9c0 pc=0x89342b +net.(*conn).Read(0xc000bdff40, {0xc002362000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0022dfaa8 sp=0xc0022dfa20 pc=0x8b176d +net.(*TCPConn).Read(0xc000bdff40, {0xc002362000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc0022dfaf0 sp=0xc0022dfaa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc000d64de0, {0xc001ebdc40, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0022dfbb0 sp=0xc0022dfaf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000d64de0}, {0xc001ebdc40, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0022dfbf8 sp=0xc0022dfbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001ebdc40, 0x9, 0x9}, {0x5b0f0a0, 0xc000d64de0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0022dfc98 sp=0xc0022dfbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001ebdc00) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0022dfdd0 sp=0xc0022dfc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00165a480, 0xc000d64e40?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0022dffb0 sp=0xc0022dfdd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0022dffe0 sp=0xc0022dffb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0022dffe8 sp=0xc0022dffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1235 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1243 [select]: +runtime.gopark(0xc0023b5e70?, 0x2?, 0x0?, 0x0?, 0xc0023b5df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0023b5c58 sp=0xc0023b5c38 pc=0x4777ae +runtime.selectgo(0xc0023b5e70, 0xc0023b5df0, 0xc0023b5e00?, 0x0, 0xc000bdff40?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0023b5da8 sp=0xc0023b5c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0007c1590, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0023b5ea0 sp=0xc0023b5da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001cf2c40) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0023b5f58 sp=0xc0023b5ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc0023b5fe0 sp=0xc0023b5f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0023b5fe8 sp=0xc0023b5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1235 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1244 [select]: +runtime.gopark(0xc0023ade48?, 0x2?, 0x0?, 0x0?, 0xc0023addcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0023adc30 sp=0xc0023adc10 pc=0x4777ae +runtime.selectgo(0xc0023ade48, 0xc0023addc8, 0xc0023addd8?, 0x0, 0xc000bdff28?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0023add80 sp=0xc0023adc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0007c1680, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0023ade78 sp=0xc0023add80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001cf2cb0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0023adf30 sp=0xc0023ade78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0023adfe0 sp=0xc0023adf30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0023adfe8 sp=0xc0023adfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1240 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1245 [select]: +runtime.gopark(0xc00210bf70?, 0x4?, 0xa9?, 0x5?, 0xc00210bdd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00210bc00 sp=0xc00210bbe0 pc=0x4777ae +runtime.selectgo(0xc00210bf70, 0xc00210bdd0, 0x5a00000000000000?, 0x0, 0x4?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00210bd50 sp=0xc00210bc00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001f4aea0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc00210bfc0 sp=0xc00210bd50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc00210bfe0 sp=0xc00210bfc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00210bfe8 sp=0xc00210bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1240 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1246 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001f077d8 sp=0xc001f077b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001f07810 sp=0xc001f077d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169dd40, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001f07830 sp=0xc001f07810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0027d5820, 0xc00236e000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001f07878 sp=0xc001f07830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0027d5800, {0xc00236e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc001f07968 sp=0xc001f07878 pc=0x50e5a5 +net.(*netFD).Read(0xc0027d5800, {0xc00236e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001f079c8 sp=0xc001f07968 pc=0x89342b +net.(*conn).Read(0xc000bdff28, {0xc00236e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc001f07a50 sp=0xc001f079c8 pc=0x8b176d +net.(*TCPConn).Read(0xc000bdff28, {0xc00236e000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc001f07a98 sp=0xc001f07a50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc000d64ea0, {0xc001ebdd20, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001f07b58 sp=0xc001f07a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc000d64ea0}, {0xc001ebdd20, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001f07ba0 sp=0xc001f07b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc001ebdd20, 0x9, 0x9}, {0x5b0f0a0, 0xc000d64ea0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001f07c40 sp=0xc001f07ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001ebdce0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001f07d78 sp=0xc001f07c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001f4aea0, 0x4208e0?, 0x7f429a46a200?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001f07ec8 sp=0xc001f07d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c1e0, {0x5b54e00?, 0xc001f4aea0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001f07f80 sp=0xc001f07ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001f07fe0 sp=0xc001f07f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001f07fe8 sp=0xc001f07fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1240 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1383 [IO wait]: +runtime.gopark(0xc0024d4fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024d4f98 sp=0xc0024d4f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0024d4fd0 sp=0xc0024d4f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c49a0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0024d4ff0 sp=0xc0024d4fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0024adea0, 0xc0022f9200?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0024d5038 sp=0xc0024d4ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0024ade80, {0xc0022f9200, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0024d5128 sp=0xc0024d5038 pc=0x50e5a5 +net.(*netFD).Read(0xc0024ade80, {0xc0022f9200, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0024d5188 sp=0xc0024d5128 pc=0x89342b +net.(*conn).Read(0xc0012c7398, {0xc0022f9200, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0024d5210 sp=0xc0024d5188 pc=0x8b176d +net.(*UnixConn).Read(0xc0012c7398, {0xc0022f9200, 0x600, 0x600}) + :1 +0x54 fp=0xc0024d5258 sp=0xc0024d5210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc00310cf18, {0xc0022f9200, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0024d52b8 sp=0xc0024d5258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000f9db28, {0x5b0f6c0, 0xc00310cf18}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0024d5320 sp=0xc0024d52b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000f9d880, {0x5b0f840?, 0xc0012c7398}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0024d5390 sp=0xc0024d5320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000f9d880, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0024d59c0 sp=0xc0024d5390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000f9d880, {0xc002810000, 0x8000, 0xc0024d5af8?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0024d5aa8 sp=0xc0024d59c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d86de0, {0xc002810000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0024d5af0 sp=0xc0024d5aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc002175680, {0xc0014f9000, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0024d5bb0 sp=0xc0024d5af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002175680}, {0xc0014f9000, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0024d5bf8 sp=0xc0024d5bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0014f9000, 0x9, 0x9}, {0x5b0f0a0, 0xc002175680}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0024d5c98 sp=0xc0024d5bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0014f8fc0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0024d5dd0 sp=0xc0024d5c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001455d40, 0xc000f9d880?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0024d5fb0 sp=0xc0024d5dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0024d5fe0 sp=0xc0024d5fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024d5fe8 sp=0xc0024d5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1380 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1251 [IO wait]: +runtime.gopark(0x4?, 0xc0017a7c30?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a7c48 sp=0xc0017a7c28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0017a7c80 sp=0xc0017a7c48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169dc48, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0017a7ca0 sp=0xc0017a7c80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001655280, 0xc002424000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0017a7ce8 sp=0xc0017a7ca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001655260, {0xc002424000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0017a7dd8 sp=0xc0017a7ce8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001d6eac0, {0xc002424000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc0017a7e38 sp=0xc0017a7dd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc0010d91d0}, {0x5b0eb40, 0xc001d6eac0}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc0017a7ee0 sp=0xc0017a7e38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc0017a7f50 sp=0xc0017a7ee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc001635e00) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc0017a7fb8 sp=0xc0017a7f50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc0017a7fe0 sp=0xc0017a7fb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a7fe8 sp=0xc0017a7fe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 1250 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 1252 [IO wait]: +runtime.gopark(0x4b05a9?, 0xc00274bc70?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00274bc48 sp=0xc00274bc28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00274bc80 sp=0xc00274bc48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169da58, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00274bca0 sp=0xc00274bc80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001655340, 0xc00244a000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00274bce8 sp=0xc00274bca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001655320, {0xc00244a000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00274bdd8 sp=0xc00274bce8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001d6eaf8, {0xc00244a000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc00274be38 sp=0xc00274bdd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc0010d91e8}, {0x5b0eb40, 0xc001d6eaf8}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc00274bee0 sp=0xc00274be38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc00274bf50 sp=0xc00274bee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc001635e60) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc00274bfb8 sp=0xc00274bf50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc00274bfe0 sp=0xc00274bfb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00274bfe8 sp=0xc00274bfe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 1250 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 1253 [select]: +runtime.gopark(0xc001de54b0?, 0x656a30?, 0x0?, 0x1?, 0xc001b6ce7c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b6cce0 sp=0xc001b6ccc0 pc=0x4777ae +runtime.selectgo(0xc001b6cf60, 0xc001b6ce78, 0x0?, 0x1, 0x40a16a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001b6ce30 sp=0xc001b6cce0 pc=0x48916b +os/exec.(*Cmd).watchCtx(0xc00116e160, 0xc001de54a0) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:757 +0x10f fp=0xc001b6cfb0 sp=0xc001b6ce30 pc=0x656a6f +os/exec.(*Cmd).Start.func4() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0x45 fp=0xc001b6cfe0 sp=0xc001b6cfb0 pc=0x6565a5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b6cfe8 sp=0xc001b6cfe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 1250 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0xf05 + +goroutine 1254 [syscall]: +syscall.Syscall6(0x0?, 0x0?, 0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/syscall/syscall_linux.go:91 +0x3d fp=0xc002103d38 sp=0xc002103cb0 pc=0x4d1b1d +os.(*Process).blockUntilWaitable(0xc001620c90) + /home/ec2-user/sdk/go1.21.0/src/os/wait_waitid.go:32 +0x9f fp=0xc002103e18 sp=0xc002103d38 pc=0x529c5f +os.(*Process).wait(0xc001620c90) + /home/ec2-user/sdk/go1.21.0/src/os/exec_unix.go:22 +0x4b fp=0xc002103e98 sp=0xc002103e18 pc=0x51e0ab +os.(*Process).Wait(...) + /home/ec2-user/sdk/go1.21.0/src/os/exec.go:134 +os/exec.(*Cmd).Wait(0xc00116e160) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:890 +0xb9 fp=0xc002103f60 sp=0xc002103e98 pc=0x657279 +github.com/onsi/gomega/gexec.(*Session).monitorForExit(0xc002377380, 0x32594b527a594f52?) + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:196 +0x3f fp=0xc002103fb0 sp=0xc002103f60 pc=0x233731f +github.com/onsi/gomega/gexec.Start.func1() + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x45 fp=0xc002103fe0 sp=0xc002103fb0 pc=0x2336d65 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002103fe8 sp=0xc002103fe0 pc=0x4ad421 +created by github.com/onsi/gomega/gexec.Start in goroutine 1250 + /home/ec2-user/go/pkg/mod/github.com/onsi/gomega@v1.27.10/gexec/session.go:96 +0x9eb + +goroutine 1258 [select]: +runtime.gopark(0xc0025a7e48?, 0x2?, 0x0?, 0x0?, 0xc0025a7dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0025a7c30 sp=0xc0025a7c10 pc=0x4777ae +runtime.selectgo(0xc0025a7e48, 0xc0025a7dc8, 0xc0025a7dd8?, 0x0, 0xc001d6eb40?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0025a7d80 sp=0xc0025a7c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001a0d400, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0025a7e78 sp=0xc0025a7d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001705110) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0025a7f30 sp=0xc0025a7e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0025a7fe0 sp=0xc0025a7f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0025a7fe8 sp=0xc0025a7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1257 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1259 [select]: +runtime.gopark(0xc00181ef70?, 0x4?, 0x20?, 0x97?, 0xc00181edd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00181ec00 sp=0xc00181ebe0 pc=0x4777ae +runtime.selectgo(0xc00181ef70, 0xc00181edd0, 0x420000c0017030c0?, 0x0, 0x5?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00181ed50 sp=0xc00181ec00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0017576c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc00181efc0 sp=0xc00181ed50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc00181efe0 sp=0xc00181efc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00181efe8 sp=0xc00181efe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1257 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1260 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00181c7d8 sp=0xc00181c7b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00181c810 sp=0xc00181c7d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169db50, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00181c830 sp=0xc00181c810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0016f92a0, 0xc002580000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00181c878 sp=0xc00181c830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0016f9280, {0xc002580000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00181c968 sp=0xc00181c878 pc=0x50e5a5 +net.(*netFD).Read(0xc0016f9280, {0xc002580000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00181c9c8 sp=0xc00181c968 pc=0x89342b +net.(*conn).Read(0xc001d6eb40, {0xc002580000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00181ca50 sp=0xc00181c9c8 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6eb40, {0xc002580000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc00181ca98 sp=0xc00181ca50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc001655ce0, {0xc0017032a0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00181cb58 sp=0xc00181ca98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001655ce0}, {0xc0017032a0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00181cba0 sp=0xc00181cb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0017032a0, 0x9, 0x9}, {0x5b0f0a0, 0xc001655ce0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00181cc40 sp=0xc00181cba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc001703260) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00181cd78 sp=0xc00181cc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0017576c0, 0x5b423c8?, 0x74736f686c000000?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc00181cec8 sp=0xc00181cd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c1e0, {0x5b54e00?, 0xc0017576c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc00181cf80 sp=0xc00181cec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc00181cfe0 sp=0xc00181cf80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00181cfe8 sp=0xc00181cfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1257 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1310 [select]: +runtime.gopark(0xc002635e70?, 0x2?, 0x0?, 0x0?, 0xc002635df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002635c58 sp=0xc002635c38 pc=0x4777ae +runtime.selectgo(0xc002635e70, 0xc002635df0, 0xc002635e00?, 0x0, 0xc00133a000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002635da8 sp=0xc002635c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001c65db0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002635ea0 sp=0xc002635da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0026547e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002635f58 sp=0xc002635ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc002635fe0 sp=0xc002635f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002635fe8 sp=0xc002635fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1353 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1314 [IO wait]: +runtime.gopark(0x4?, 0xc000d02c30?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc000d02c48 sp=0xc000d02c28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc000d02c80 sp=0xc000d02c48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d770, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc000d02ca0 sp=0xc000d02c80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00128a7a0, 0xc00249a000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc000d02ce8 sp=0xc000d02ca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00128a780, {0xc00249a000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc000d02dd8 sp=0xc000d02ce8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc0012c6dc0, {0xc00249a000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc000d02e38 sp=0xc000d02dd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc0013200c0}, {0x5b0eb40, 0xc0012c6dc0}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc000d02ee0 sp=0xc000d02e38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc000d02f50 sp=0xc000d02ee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc00127dcc0) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc000d02fb8 sp=0xc000d02f50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc000d02fe0 sp=0xc000d02fb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000d02fe8 sp=0xc000d02fe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 1250 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 1355 [chan receive]: +runtime.gopark(0xc001f48830?, 0xffffffff?, 0xa9?, 0x5?, 0xc001ef3f40?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001ef3f08 sp=0xc001ef3ee8 pc=0x4777ae +runtime.chanrecv(0xc001089680, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001ef3f80 sp=0xc001ef3f08 pc=0x441325 +runtime.chanrecv1(0xc001b6e730?, 0xc001ef3fd0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001ef3fa8 sp=0xc001ef3f80 pc=0x440f72 +github.com/rancher/opni/pkg/clients.NewGatewayClient.func1() + /home/ec2-user/opni/pkg/clients/gateway_client.go:69 +0x4a fp=0xc001ef3fe0 sp=0xc001ef3fa8 pc=0x200c96a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001ef3fe8 sp=0xc001ef3fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/clients.NewGatewayClient in goroutine 1320 + /home/ec2-user/opni/pkg/clients/gateway_client.go:68 +0x2dd + +goroutine 1379 [select]: +runtime.gopark(0xc0024e1f70?, 0x2?, 0xb4?, 0x1b?, 0xc0024e1ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024e1d20 sp=0xc0024e1d00 pc=0x4777ae +runtime.selectgo(0xc0024e1f70, 0xc0024e1eb8, 0x0?, 0x0, 0xc0001261c8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0024e1e70 sp=0xc0024e1d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc001d86ae0, {0x5b426e0, 0xc002338cd0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0024e1fa0 sp=0xc0024e1e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0024e1fe0 sp=0xc0024e1fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024e1fe8 sp=0xc0024e1fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1361 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1316 [select]: +runtime.gopark(0xc00123e790?, 0x656a30?, 0x0?, 0x1?, 0xc001983e7c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001983ce0 sp=0xc001983cc0 pc=0x4777ae +runtime.selectgo(0xc001983f60, 0xc001983e78, 0x4b05a9?, 0x1, 0x1f76a9f?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001983e30 sp=0xc001983ce0 pc=0x48916b +os/exec.(*Cmd).watchCtx(0xc0026c3340, 0xc00123e780) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:757 +0x10f fp=0xc001983fb0 sp=0xc001983e30 pc=0x656a6f +os/exec.(*Cmd).Start.func4() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0x45 fp=0xc001983fe0 sp=0xc001983fb0 pc=0x6565a5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001983fe8 sp=0xc001983fe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 1250 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:743 +0xf05 + +goroutine 1315 [IO wait]: +runtime.gopark(0x4b05a9?, 0xc002100c70?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002100c48 sp=0xc002100c28 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002100c80 sp=0xc002100c48 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d488, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002100ca0 sp=0xc002100c80 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00128a860, 0xc002610000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002100ce8 sp=0xc002100ca0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00128a840, {0xc002610000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc002100dd8 sp=0xc002100ce8 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc0012c6e00, {0xc002610000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc002100e38 sp=0xc002100dd8 pc=0x51ed0d +io.copyBuffer({0x5b0ede0, 0xc0013200d8}, {0x5b0eb40, 0xc0012c6e00}, {0x0, 0x0, 0x0}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:430 +0x27e fp=0xc002100ee0 sp=0xc002100e38 pc=0x50115e +io.Copy(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:389 +os/exec.(*Cmd).writerDescriptor.func1() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:560 +0x5b fp=0xc002100f50 sp=0xc002100ee0 pc=0x6552db +os/exec.(*Cmd).Start.func2(0xc00127dd60) + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:717 +0x3d fp=0xc002100fb8 sp=0xc002100f50 pc=0x65669d +os/exec.(*Cmd).Start.func3() + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:729 +0x42 fp=0xc002100fe0 sp=0xc002100fb8 pc=0x656622 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002100fe8 sp=0xc002100fe0 pc=0x4ad421 +created by os/exec.(*Cmd).Start in goroutine 1250 + /home/ec2-user/sdk/go1.21.0/src/os/exec/exec.go:716 +0xf3a + +goroutine 1446 [select]: +runtime.gopark(0xc001086fb0?, 0x2?, 0xb4?, 0x1b?, 0xc001086f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001086de8 sp=0xc001086dc8 pc=0x4777ae +runtime.selectgo(0xc001086fb0, 0xc001086f70, 0x0?, 0x0, 0x6000103?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001086f38 sp=0xc001086de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc001086fe0 sp=0xc001086f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001086fe8 sp=0xc001086fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1361 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1389 [select]: +runtime.gopark(0xc00260ffb0?, 0x2?, 0xb0?, 0xfd?, 0xc00260fee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00260fd58 sp=0xc00260fd38 pc=0x4777ae +runtime.selectgo(0xc00260ffb0, 0xc00260fee0, 0xc00260ff98?, 0x0, 0xc0027107b0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00260fea8 sp=0xc00260fd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc00260ffe0 sp=0xc00260fea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00260ffe8 sp=0xc00260ffe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1444 [select]: +runtime.gopark(0xc001e22890?, 0x2?, 0x98?, 0x27?, 0xc001e2283c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e226a8 sp=0xc001e22688 pc=0x4777ae +runtime.selectgo(0xc001e22890, 0xc001e22838, 0x7f429a489100?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e227f8 sp=0xc001e226a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002658460, {0xc002652c70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001e228c0 sp=0xc001e227f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002658460, {0xc002652c70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001e22958 sp=0xc001e228c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002652bd0, {0xc002652c70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001e229a0 sp=0xc001e22958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002652bd0}, {0xc002652c70, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001e229e8 sp=0xc001e229a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0018f4fc0, {0xc002652c70, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001e22a58 sp=0xc001e229e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002652c60, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001e22b28 sp=0xc001e22a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc002171500?, 0xc0018f4fc0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001e22c18 sp=0xc001e22b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002652c60, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc002658780}, 0xc0025ccf60?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001e22cd0 sp=0xc001e22c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000b9c0f0, {0x43dcf60?, 0xc002658780}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001e22f08 sp=0xc001e22cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002652db0, {0x43dcf60, 0xc002658780}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001e22ff0 sp=0xc001e22f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc0015ffd20, {0x43dcf60, 0xc002658780}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001e23028 sp=0xc001e22ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00271e250) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001e23068 sp=0xc001e23028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00271e250?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001e230a8 sp=0xc001e23068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00271e260) + :1 +0x46 fp=0xc001e230e0 sp=0xc001e230a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0025cd0e0, {0x5b51368, 0xc00271e260}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001e23320 sp=0xc001e230e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00234fbc0, {0x5b51038, 0xc00271e250}) + :1 +0xaf fp=0xc001e23370 sp=0xc001e23320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00234fbc0}, {0x5b4a6f0?, 0xc0015ffd20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001e233b0 sp=0xc001e23370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc002652db0}, 0xc000580d38?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001e23400 sp=0xc001e233b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc002652db0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001e23490 sp=0xc001e23400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc000b9c0f0}, 0xc000580d38, 0xc002170c40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001e237d0 sp=0xc001e23490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc000b9c0f0}, 0x3f252a0?, 0xc00271e180?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001e23848 sp=0xc001e237d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc0018f4fc0, 0xc0025cd170, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001e23db0 sp=0xc001e23848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc0018f4fc0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001e23f28 sp=0xc001e23db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001e23fe0 sp=0xc001e23f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e23fe8 sp=0xc001e23fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1387 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1312 [select]: +runtime.gopark(0xc0015e7fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0015e7f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015e7de8 sp=0xc0015e7dc8 pc=0x4777ae +runtime.selectgo(0xc0015e7fb0, 0xc0015e7f70, 0x0?, 0x0, 0xc0026a43c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0015e7f38 sp=0xc0015e7de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0015e7fe0 sp=0xc0015e7f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015e7fe8 sp=0xc0015e7fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1361 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1390 [chan receive]: +runtime.gopark(0x40a16a0?, 0x0?, 0x40?, 0x6e?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002101d60 sp=0xc002101d40 pc=0x4777ae +runtime.chanrecv(0xc0022fb260, 0xc002101e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002101dd8 sp=0xc002101d60 pc=0x441325 +runtime.chanrecv1(0xc002339618?, 0x40a16a0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002101e00 sp=0xc002101dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002101fe0 sp=0xc002101e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002101fe8 sp=0xc002101fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1384 [select]: +runtime.gopark(0xc002987e70?, 0x2?, 0x0?, 0x0?, 0xc002987df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002987c58 sp=0xc002987c38 pc=0x4777ae +runtime.selectgo(0xc002987e70, 0xc002987df0, 0xc002987e00?, 0x0, 0xc000f9d880?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002987da8 sp=0xc002987c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0023394f0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002987ea0 sp=0xc002987da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002714a80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002987f58 sp=0xc002987ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc002987fe0 sp=0xc002987f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002987fe8 sp=0xc002987fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1380 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1385 [select]: +runtime.gopark(0xc002879e48?, 0x2?, 0x0?, 0x0?, 0xc002879dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002879c30 sp=0xc002879c10 pc=0x4777ae +runtime.selectgo(0xc002879e48, 0xc002879dc8, 0xc002879dd8?, 0x0, 0xc00133b180?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002879d80 sp=0xc002879c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0023395e0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002879e78 sp=0xc002879d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002714bd0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002879f30 sp=0xc002879e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002879fe0 sp=0xc002879f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002879fe8 sp=0xc002879fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1394 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1386 [select]: +runtime.gopark(0xc001987f70?, 0x4?, 0x10?, 0x6d?, 0xc001987dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001987c00 sp=0xc001987be0 pc=0x4777ae +runtime.selectgo(0xc001987f70, 0xc001987dd0, 0xc001987d80?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001987d50 sp=0xc001987c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0024e7860) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001987fc0 sp=0xc001987d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001987fe0 sp=0xc001987fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001987fe8 sp=0xc001987fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1394 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1313 [select]: +runtime.gopark(0xc001e3afb0?, 0x2?, 0xb0?, 0xad?, 0xc001e3aee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001e3ad58 sp=0xc001e3ad38 pc=0x4777ae +runtime.selectgo(0xc001e3afb0, 0xc001e3aee0, 0x1128d01?, 0x0, 0xffffffffffffffff?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001e3aea8 sp=0xc001e3ad58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc001e3afe0 sp=0xc001e3aea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001e3afe8 sp=0xc001e3afe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1357 [select]: +runtime.gopark(0xc001d49f48?, 0x2?, 0x50?, 0x9d?, 0xc001d49f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001d49d80 sp=0xc001d49d60 pc=0x4777ae +runtime.selectgo(0xc001d49f48, 0xc001d49f00, 0xc001d49f08?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001d49ed0 sp=0xc001d49d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).Recv(0xc0025cd230) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:124 +0xeb fp=0xc001d49f78 sp=0xc001d49ed0 pc=0x2154feb +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc001abbc70) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc001d49fc0 sp=0xc001d49f78 pc=0x215759f +github.com/hashicorp/go-plugin.(*GRPCServer).Init.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x34 fp=0xc001d49fe0 sp=0xc001d49fc0 pc=0x215adf4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001d49fe8 sp=0xc001d49fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*GRPCServer).Init in goroutine 1356 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x89d + +goroutine 1358 [IO wait]: +runtime.gopark(0x2?, 0x48eaa0?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00215eb30 sp=0xc00215eb10 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00215eb68 sp=0xc00215eb30 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d1a0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00215eb88 sp=0xc00215eb68 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001e54fe0, 0xc00271a000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00215ebd0 sp=0xc00215eb88 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001e54fc0, {0xc00271a000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00215ecc0 sp=0xc00215ebd0 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001f4e390, {0xc00271a000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc00215ed20 sp=0xc00215ecc0 pc=0x51ed0d +io.(*teeReader).Read(0xc002180940, {0xc00271a000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:620 +0x5d fp=0xc00215ed80 sp=0xc00215ed20 pc=0x501c5d +bufio.(*Reader).Read(0xc00215ef18, {0xc0024cf800, 0x400, 0x400}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00215ee40 sp=0xc00215ed80 pc=0x5f943e +github.com/hashicorp/go-plugin.copyChan({0x5b611c8, 0xc0025c2c60}, 0xc001d9d440?, {0x5b0ee40?, 0xc002180940}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:184 +0x1f4 fp=0xc00215ef80 sp=0xc00215ee40 pc=0x215ccf4 +github.com/hashicorp/go-plugin.newGRPCStdioServer.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:40 +0x6c fp=0xc00215efe0 sp=0xc00215ef80 pc=0x215b9ac +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00215efe8 sp=0xc00215efe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCStdioServer in goroutine 1356 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:40 +0x169 + +goroutine 1359 [IO wait]: +runtime.gopark(0x4b05a9?, 0xc00260bb60?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00260bb30 sp=0xc00260bb10 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00260bb68 sp=0xc00260bb30 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c5068, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00260bb88 sp=0xc00260bb68 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001e550a0, 0xc0024b9000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00260bbd0 sp=0xc00260bb88 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001e55080, {0xc0024b9000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00260bcc0 sp=0xc00260bbd0 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001f4e3a0, {0xc0024b9000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc00260bd20 sp=0xc00260bcc0 pc=0x51ed0d +io.(*teeReader).Read(0xc002180960, {0xc0024b9000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:620 +0x5d fp=0xc00260bd80 sp=0xc00260bd20 pc=0x501c5d +bufio.(*Reader).Read(0xc00260bf18, {0xc0024cf400, 0x400, 0x400}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00260be40 sp=0xc00260bd80 pc=0x5f943e +github.com/hashicorp/go-plugin.copyChan({0x5b611c8, 0xc0025c2c60}, 0xc001d9d4a0?, {0x5b0ee40?, 0xc002180960}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:184 +0x1f4 fp=0xc00260bf80 sp=0xc00260be40 pc=0x215ccf4 +github.com/hashicorp/go-plugin.newGRPCStdioServer.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:41 +0x6c fp=0xc00260bfe0 sp=0xc00260bf80 pc=0x215b90c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00260bfe8 sp=0xc00260bfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCStdioServer in goroutine 1356 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:41 +0x265 + +goroutine 1360 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024cd940 sp=0xc0024cd920 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0024cd978 sp=0xc0024cd940 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d678, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0024cd998 sp=0xc0024cd978 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0025f2320, 0xc0024cda01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0024cd9e0 sp=0xc0024cd998 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0025f2300) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0024cdad0 sp=0xc0024cd9e0 pc=0x513f65 +net.(*netFD).accept(0xc0025f2300) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0024cdbc8 sp=0xc0024cdad0 pc=0x8967be +net.(*UnixListener).accept(0xc0025cce70) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc0024cdc08 sp=0xc0024cdbc8 pc=0x8cedfe +net.(*UnixListener).Accept(0xc0025cce70) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc0024cdc78 sp=0xc0024cdc08 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc002180920) + :1 +0x44 fp=0xc0024cdcb0 sp=0xc0024cdc78 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc0018534a0, {0x5b2c6a0, 0xc002180920}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0024cdee0 sp=0xc0024cdcb0 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCServer).Serve(0xc0025c2d80, {0x5b2c6a0, 0xc002180920}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:153 +0xdd fp=0xc0024cdf98 sp=0xc0024cdee0 pc=0x215b33d +github.com/hashicorp/go-plugin.Serve.func6() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x62 fp=0xc0024cdfe0 sp=0xc0024cdf98 pc=0x2166b62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024cdfe8 sp=0xc0024cdfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.Serve in goroutine 1356 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x1f67 + +goroutine 1453 [select]: +runtime.gopark(0xc002666310?, 0x13b0513?, 0x0?, 0x1?, 0xc0029b1ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b1d30 sp=0xc0029b1d10 pc=0x4777ae +runtime.selectgo(0xc0029b1f60, 0xc0029b1ec8, 0xc0029b1eb8?, 0x1, 0x28?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029b1e80 sp=0xc0029b1d30 pc=0x48916b +github.com/lestrrat-go/backoff/v2.(*controller).loop(0xc00251d020) + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:64 +0x318 fp=0xc0029b1fc0 sp=0xc0029b1e80 pc=0x13b0538 +github.com/lestrrat-go/backoff/v2.newController.func1() + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x34 fp=0xc0029b1fe0 sp=0xc0029b1fc0 pc=0x13b01f4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b1fe8 sp=0xc0029b1fe0 pc=0x4ad421 +created by github.com/lestrrat-go/backoff/v2.newController in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/lestrrat-go/backoff/v2@v2.0.8/controller.go:46 +0x4dd + +goroutine 1362 [select]: +runtime.gopark(0xc0029e7da8?, 0x2?, 0x60?, 0x78?, 0xc0029e7a94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029e7818 sp=0xc0029e77f8 pc=0x4777ae +runtime.selectgo(0xc0029e7da8, 0xc0029e7a90, 0x0?, 0x0, 0x1860a0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029e7968 sp=0xc0029e7818 pc=0x48916b +github.com/hashicorp/go-plugin.Serve(0xc001e55500) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:474 +0x2091 fp=0xc0029e7fc0 sp=0xc0029e7968 pc=0x21665b1 +github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins.func4() + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x34 fp=0xc0029e7fe0 sp=0xc0029e7fc0 pc=0x24cd234 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029e7fe8 sp=0xc0029e7fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/test.TestPluginSet.LoadPlugins in goroutine 1320 + /home/ec2-user/opni/pkg/test/plugin.go:133 +0x76b + +goroutine 1294 [select]: +runtime.gopark(0xc0025eb748?, 0x2?, 0xd0?, 0xb5?, 0xc0025eb6fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0025eb578 sp=0xc0025eb558 pc=0x4777ae +runtime.selectgo(0xc0025eb748, 0xc0025eb6f8, 0xc0021827c0?, 0x0, 0xc00257d798?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0025eb6c8 sp=0xc0025eb578 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c6b200) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc0025eb778 sp=0xc0025eb6c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc0026a88f0, {0x43dd860?, 0xc0028460a0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0025eb9d8 sp=0xc0025eb778 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001d569b4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc0025eba38 sp=0xc0025eb9d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001d56900, 0xc0025ebbf0, 0xc0025ebbc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc0025ebae0 sp=0xc0025eba38 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d56900, {0x43dd860, 0xc0028460a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc0025ebc20 sp=0xc0025ebae0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0021e6200, {0x43dd860, 0xc0028460a0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc0025ebde8 sp=0xc0025ebc20 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamClient).Recv(0xc0025771b0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:132 +0x66 fp=0xc0025ebe28 sp=0xc0025ebde8 pc=0x2102606 +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream(0xc001ff19c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:197 +0x2e8 fp=0xc0025ebfc0 sp=0xc0025ebe28 pc=0x2155468 +github.com/hashicorp/go-plugin.newGRPCClient.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x34 fp=0xc0025ebfe0 sp=0xc0025ebfc0 pc=0x2159954 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0025ebfe8 sp=0xc0025ebfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 1361 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x62e + +goroutine 1295 [select]: +runtime.gopark(0xc002795e48?, 0x2?, 0x0?, 0x0?, 0xc002795dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002795c30 sp=0xc002795c10 pc=0x4777ae +runtime.selectgo(0xc002795e48, 0xc002795dc8, 0xc00257b7a0?, 0x0, 0x5?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002795d80 sp=0xc002795c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0026a4e10, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002795e78 sp=0xc002795d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00257b7a0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002795f30 sp=0xc002795e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002795fe0 sp=0xc002795f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002795fe8 sp=0xc002795fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1344 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1296 [select]: +runtime.gopark(0xc001820f70?, 0x4?, 0xa9?, 0x5?, 0xc001820dd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001820c00 sp=0xc001820be0 pc=0x4777ae +runtime.selectgo(0xc001820f70, 0xc001820dd0, 0x40a16a0?, 0x0, 0xc001fafff0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001820d50 sp=0xc001820c00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002515040) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc001820fc0 sp=0xc001820d50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc001820fe0 sp=0xc001820fc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001820fe8 sp=0xc001820fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1344 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1297 [IO wait]: +runtime.gopark(0xc0024dd048?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024dd000 sp=0xc0024dcfe0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0024dd038 sp=0xc0024dd000 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d298, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0024dd058 sp=0xc0024dd038 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0024adba0, 0xc002e2ea00?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0024dd0a0 sp=0xc0024dd058 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0024adb80, {0xc002e2ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0024dd190 sp=0xc0024dd0a0 pc=0x50e5a5 +net.(*netFD).Read(0xc0024adb80, {0xc002e2ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0024dd1f0 sp=0xc0024dd190 pc=0x89342b +net.(*conn).Read(0xc0012c72e0, {0xc002e2ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0024dd278 sp=0xc0024dd1f0 pc=0x8b176d +net.(*TCPConn).Read(0xc0012c72e0, {0xc002e2ea00, 0x500, 0x500}) + :1 +0x54 fp=0xc0024dd2c0 sp=0xc0024dd278 pc=0x8d72f4 +crypto/tls.(*atLeastReader).Read(0xc00304a2a0, {0xc002e2ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0024dd320 sp=0xc0024dd2c0 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc000f9d428, {0x5b0f6c0, 0xc00304a2a0}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0024dd388 sp=0xc0024dd320 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc000f9d180, {0x5b0f820?, 0xc0012c72e0}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0024dd3f8 sp=0xc0024dd388 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc000f9d180, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0024dda28 sp=0xc0024dd3f8 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc000f9d180, {0xc002686580, 0x9, 0x4aef8c?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0024ddb10 sp=0xc0024dda28 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0026e0240, {0xc002686580, 0x9, 0x9}) + :1 +0x6d fp=0xc0024ddb58 sp=0xc0024ddb10 pc=0x10969ed +io.ReadAtLeast({0x7f429a510838, 0xc0026e0240}, {0xc002686580, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0024ddba0 sp=0xc0024ddb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc002686580, 0x9, 0x9}, {0x7f429a510838, 0xc0026e0240}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0024ddc40 sp=0xc0024ddba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc002686540) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0024ddd78 sp=0xc0024ddc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002515040, 0x2020200a3936202d?, 0x200a313131202d20?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc0024ddec8 sp=0xc0024ddd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00170e000, {0x5b54e00?, 0xc002515040}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc0024ddf80 sp=0xc0024ddec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc0024ddfe0 sp=0xc0024ddf80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024ddfe8 sp=0xc0024ddfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1344 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1363 [select]: +runtime.gopark(0xc001487f48?, 0x2?, 0xc8?, 0xf1?, 0xc001487f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001487d80 sp=0xc001487d60 pc=0x4777ae +runtime.selectgo(0xc001487f48, 0xc001487f00, 0xc001487f08?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001487ed0 sp=0xc001487d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).Recv(0xc0025cdda0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:124 +0xeb fp=0xc001487f78 sp=0xc001487ed0 pc=0x2154feb +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc0025fa0f0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc001487fc0 sp=0xc001487f78 pc=0x215759f +github.com/hashicorp/go-plugin.(*GRPCServer).Init.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x34 fp=0xc001487fe0 sp=0xc001487fc0 pc=0x215adf4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001487fe8 sp=0xc001487fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*GRPCServer).Init in goroutine 1362 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:88 +0x89d + +goroutine 1364 [IO wait]: +runtime.gopark(0x2?, 0x48eaa0?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002104b30 sp=0xc002104b10 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002104b68 sp=0xc002104b30 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c4d80, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002104b88 sp=0xc002104b68 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001e55880, 0xc002788000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002104bd0 sp=0xc002104b88 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001e55860, {0xc002788000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc002104cc0 sp=0xc002104bd0 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001f4e478, {0xc002788000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc002104d20 sp=0xc002104cc0 pc=0x51ed0d +io.(*teeReader).Read(0xc002180d40, {0xc002788000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:620 +0x5d fp=0xc002104d80 sp=0xc002104d20 pc=0x501c5d +bufio.(*Reader).Read(0xc002104f18, {0xc0025fd000, 0x400, 0x400}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002104e40 sp=0xc002104d80 pc=0x5f943e +github.com/hashicorp/go-plugin.copyChan({0x5b611c8, 0xc0025c2ea0}, 0xc001d9d800?, {0x5b0ee40?, 0xc002180d40}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:184 +0x1f4 fp=0xc002104f80 sp=0xc002104e40 pc=0x215ccf4 +github.com/hashicorp/go-plugin.newGRPCStdioServer.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:40 +0x6c fp=0xc002104fe0 sp=0xc002104f80 pc=0x215b9ac +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002104fe8 sp=0xc002104fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCStdioServer in goroutine 1362 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:40 +0x169 + +goroutine 1365 [IO wait]: +runtime.gopark(0x10?, 0xc001fab200?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00234cb30 sp=0xc00234cb10 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00234cb68 sp=0xc00234cb30 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f424169d390, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00234cb88 sp=0xc00234cb68 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc001e55940, 0xc002789000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00234cbd0 sp=0xc00234cb88 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc001e55920, {0xc002789000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00234ccc0 sp=0xc00234cbd0 pc=0x50e5a5 +os.(*File).read(...) + /home/ec2-user/sdk/go1.21.0/src/os/file_posix.go:29 +os.(*File).Read(0xc001f4e488, {0xc002789000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/os/file.go:118 +0xad fp=0xc00234cd20 sp=0xc00234ccc0 pc=0x51ed0d +io.(*teeReader).Read(0xc002180d60, {0xc002789000, 0x1000, 0x1000}) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:620 +0x5d fp=0xc00234cd80 sp=0xc00234cd20 pc=0x501c5d +bufio.(*Reader).Read(0xc00234cf18, {0xc0025fd400, 0x400, 0x400}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00234ce40 sp=0xc00234cd80 pc=0x5f943e +github.com/hashicorp/go-plugin.copyChan({0x5b611c8, 0xc0025c2ea0}, 0xc001d9d860?, {0x5b0ee40?, 0xc002180d60}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:184 +0x1f4 fp=0xc00234cf80 sp=0xc00234ce40 pc=0x215ccf4 +github.com/hashicorp/go-plugin.newGRPCStdioServer.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:41 +0x6c fp=0xc00234cfe0 sp=0xc00234cf80 pc=0x215b90c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00234cfe8 sp=0xc00234cfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCStdioServer in goroutine 1362 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:41 +0x265 + +goroutine 1366 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0022dd940 sp=0xc0022dd920 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0022dd978 sp=0xc0022dd940 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c4e78, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0022dd998 sp=0xc0022dd978 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0025f2620, 0xc0022dda01?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0022dd9e0 sp=0xc0022dd998 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0025f2600) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc0022ddad0 sp=0xc0022dd9e0 pc=0x513f65 +net.(*netFD).accept(0xc0025f2600) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc0022ddbc8 sp=0xc0022ddad0 pc=0x8967be +net.(*UnixListener).accept(0xc0025cd9e0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock_posix.go:172 +0x3e fp=0xc0022ddc08 sp=0xc0022ddbc8 pc=0x8cedfe +net.(*UnixListener).Accept(0xc0025cd9e0) + /home/ec2-user/sdk/go1.21.0/src/net/unixsock.go:260 +0x65 fp=0xc0022ddc78 sp=0xc0022ddc08 pc=0x8cc7e5 +github.com/hashicorp/go-plugin.(*rmListener).Accept(0xc002180d20) + :1 +0x44 fp=0xc0022ddcb0 sp=0xc0022ddc78 pc=0x2168444 +google.golang.org/grpc.(*Server).Serve(0xc001853680, {0x5b2c6a0, 0xc002180d20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:844 +0x5be fp=0xc0022ddee0 sp=0xc0022ddcb0 pc=0x117789e +github.com/hashicorp/go-plugin.(*GRPCServer).Serve(0xc0025c2fc0, {0x5b2c6a0, 0xc002180d20}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_server.go:153 +0xdd fp=0xc0022ddf98 sp=0xc0022ddee0 pc=0x215b33d +github.com/hashicorp/go-plugin.Serve.func6() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x62 fp=0xc0022ddfe0 sp=0xc0022ddf98 pc=0x2166b62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0022ddfe8 sp=0xc0022ddfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.Serve in goroutine 1362 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/server.go:468 +0x1f67 + +goroutine 1477 [select]: +runtime.gopark(0xc002c0ced8?, 0x2?, 0xb4?, 0x1b?, 0xc002c0ce84?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c0ccf8 sp=0xc002c0ccd8 pc=0x4777ae +runtime.selectgo(0xc002c0ced8, 0xc002c0ce80, 0xf?, 0x0, 0x8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c0ce48 sp=0xc002c0ccf8 pc=0x48916b +github.com/rancher/opni/pkg/util.ServeHandler({0x5b426e0, 0xc002c2e550}, {0x5b13c00?, 0xc002514340}, {0x5b2a6d0?, 0xc002bf8b80}) + /home/ec2-user/opni/pkg/util/servers.go:21 +0x26a fp=0xc002c0cf08 sp=0xc002c0ce48 pc=0x176a94a +github.com/rancher/opni/pkg/agent/v2.(*Agent).ListenAndServe.func2() + /home/ec2-user/opni/pkg/agent/v2/agent.go:442 +0x1b9 fp=0xc002c0cfa0 sp=0xc002c0cf08 pc=0x21bf359 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc002c0cfe0 sp=0xc002c0cfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c0cfe8 sp=0xc002c0cfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1320 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1368 [chan receive]: +runtime.gopark(0xc0025fa558?, 0x7c5b129634f4f?, 0x0?, 0x80?, 0xc000072780?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015e5d10 sp=0xc0015e5cf0 pc=0x4777ae +runtime.chanrecv(0xc001e55c80, 0xc0015e5e40, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0015e5d88 sp=0xc0015e5d10 pc=0x441325 +runtime.chanrecv2(0x3b9aca00?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc0015e5db0 sp=0xc0015e5d88 pc=0x440f92 +github.com/hashicorp/go-plugin.pidWait(0x1860a0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/process.go:20 +0xdb fp=0xc0015e5e68 sp=0xc0015e5db0 pc=0x216033b +github.com/hashicorp/go-plugin.(*Client).reattach.func1(0xc000b3d0e0?) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:853 +0xd8 fp=0xc0015e5fb8 sp=0xc0015e5e68 pc=0x21528b8 +github.com/hashicorp/go-plugin.(*Client).reattach.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:862 +0x42 fp=0xc0015e5fe0 sp=0xc0015e5fb8 pc=0x21527a2 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015e5fe8 sp=0xc0015e5fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*Client).reattach in goroutine 1367 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/client.go:846 +0x365 + +goroutine 1369 [select]: +runtime.gopark(0xc0024dbf70?, 0x2?, 0xb4?, 0x1b?, 0xc0024dbebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024dbd20 sp=0xc0024dbd00 pc=0x4777ae +runtime.selectgo(0xc0024dbf70, 0xc0024dbeb8, 0xc0027a2a00?, 0x0, 0xc0027a2b00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0024dbe70 sp=0xc0024dbd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002181200, {0x5b426e0, 0xc0025fa370}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0024dbfa0 sp=0xc0024dbe70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0024dbfe0 sp=0xc0024dbfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024dbfe8 sp=0xc0024dbfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1367 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1370 [select]: +runtime.gopark(0xc0027c9f70?, 0x2?, 0xb4?, 0x1b?, 0xc0027c9ebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027c9d20 sp=0xc0027c9d00 pc=0x4777ae +runtime.selectgo(0xc0027c9f70, 0xc0027c9eb8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0027c9e70 sp=0xc0027c9d20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002181260, {0x5b426e0, 0xc0025fa3c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc0027c9fa0 sp=0xc0027c9e70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc0027c9fe0 sp=0xc0027c9fa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0027c9fe8 sp=0xc0027c9fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1367 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1399 [select]: +runtime.gopark(0xc0029b5fb0?, 0x2?, 0xb4?, 0x1b?, 0xc0029b5f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b5de8 sp=0xc0029b5dc8 pc=0x4777ae +runtime.selectgo(0xc0029b5fb0, 0xc0029b5f70, 0xc0029b5fd0?, 0x0, 0xc002715570?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029b5f38 sp=0xc0029b5de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc0029b5fe0 sp=0xc0029b5f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b5fe8 sp=0xc0029b5fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1376 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1400 [select]: +runtime.gopark(0xc0015e6fb0?, 0x2?, 0xb0?, 0x6d?, 0xc0015e6ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0015e6d58 sp=0xc0015e6d38 pc=0x4777ae +runtime.selectgo(0xc0015e6fb0, 0xc0015e6ee0, 0xc0015e6f98?, 0x0, 0xc002780600?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0015e6ea8 sp=0xc0015e6d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0015e6fe0 sp=0xc0015e6ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0015e6fe8 sp=0xc0015e6fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1376 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1420 [IO wait]: +runtime.gopark(0xc0027c4fe0?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027c4f98 sp=0xc0027c4f78 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc0027c4fd0 sp=0xc0027c4f98 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c4a98, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc0027c4ff0 sp=0xc0027c4fd0 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc0025f2ba0, 0xc00284a000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0027c5038 sp=0xc0027c4ff0 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc0025f2b80, {0xc00284a000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0027c5128 sp=0xc0027c5038 pc=0x50e5a5 +net.(*netFD).Read(0xc0025f2b80, {0xc00284a000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc0027c5188 sp=0xc0027c5128 pc=0x89342b +net.(*conn).Read(0xc001f4e568, {0xc00284a000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0027c5210 sp=0xc0027c5188 pc=0x8b176d +net.(*UnixConn).Read(0xc001f4e568, {0xc00284a000, 0x600, 0x600}) + :1 +0x54 fp=0xc0027c5258 sp=0xc0027c5210 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0030c4510, {0xc00284a000, 0x600, 0x600}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc0027c52b8 sp=0xc0027c5258 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00125fea8, {0x5b0f6c0, 0xc0030c4510}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0027c5320 sp=0xc0027c52b8 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00125fc00, {0x5b0f840?, 0xc001f4e568}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc0027c5390 sp=0xc0027c5320 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00125fc00, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc0027c59c0 sp=0xc0027c5390 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00125fc00, {0xc002bb4000, 0x8000, 0xc0027c5af8?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc0027c5aa8 sp=0xc0027c59c0 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc001d873c0, {0xc002bb4000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc0027c5af0 sp=0xc0027c5aa8 pc=0x10969ed +bufio.(*Reader).Read(0xc00245eae0, {0xc0014f91c0, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc0027c5bb0 sp=0xc0027c5af0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00245eae0}, {0xc0014f91c0, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0027c5bf8 sp=0xc0027c5bb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc0014f91c0, 0x9, 0x9}, {0x5b0f0a0, 0xc00245eae0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc0027c5c98 sp=0xc0027c5bf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0014f9180) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc0027c5dd0 sp=0xc0027c5c98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001d846c0, 0xc000813330?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc0027c5fb0 sp=0xc0027c5dd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc0027c5fe0 sp=0xc0027c5fb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0027c5fe8 sp=0xc0027c5fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1372 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1375 [select]: +runtime.gopark(0xc001bc3f48?, 0x2?, 0x2a?, 0xce?, 0xc001bc3f04?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bc3d80 sp=0xc001bc3d60 pc=0x4777ae +runtime.selectgo(0xc001bc3f48, 0xc001bc3f00, 0xc001bc3f08?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bc3ed0 sp=0xc001bc3d80 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).Recv(0xc001ab9c40) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:234 +0xeb fp=0xc001bc3f78 sp=0xc001bc3ed0 pc=0x2155d2b +github.com/hashicorp/go-plugin.(*GRPCBroker).Run(0xc0025fa8c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:414 +0x5f fp=0xc001bc3fc0 sp=0xc001bc3f78 pc=0x215759f +github.com/hashicorp/go-plugin.newGRPCClient.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x34 fp=0xc001bc3fe0 sp=0xc001bc3fc0 pc=0x21599b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bc3fe8 sp=0xc001bc3fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 1367 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:67 +0x5b7 + +goroutine 1376 [select]: +runtime.gopark(0xc0022f7748?, 0x2?, 0xd0?, 0x75?, 0xc0022f76fc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0022f7578 sp=0xc0022f7558 pc=0x4777ae +runtime.selectgo(0xc0022f7748, 0xc0022f76f8, 0xc0020467c0?, 0x0, 0xc0025f2f18?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0022f76c8 sp=0xc0022f7578 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001d56ea0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc0022f7778 sp=0xc0022f76c8 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002782d00, {0x43dd860?, 0xc0026a5400}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0022f79d8 sp=0xc0022f7778 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0019cad14?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc0022f7a38 sp=0xc0022f79d8 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0019cac60, 0xc0022f7bf0, 0xc0022f7bc0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc0022f7ae0 sp=0xc0022f7a38 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019cac60, {0x43dd860, 0xc0026a5400}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc0022f7c20 sp=0xc0022f7ae0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc001ff1e00, {0x43dd860, 0xc0026a5400}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc0022f7de8 sp=0xc0022f7c20 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamClient).Recv(0xc0026882c0) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:132 +0x66 fp=0xc0022f7e28 sp=0xc0022f7de8 pc=0x2102606 +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream(0xc001ab9c40) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:197 +0x2e8 fp=0xc0022f7fc0 sp=0xc0022f7e28 pc=0x2155468 +github.com/hashicorp/go-plugin.newGRPCClient.func2() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x34 fp=0xc0022f7fe0 sp=0xc0022f7fc0 pc=0x2159954 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0022f7fe8 sp=0xc0022f7fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 1367 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:68 +0x62e + +goroutine 1401 [chan receive]: +runtime.gopark(0x40a16a0?, 0x0?, 0x10?, 0x81?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002b9fd60 sp=0xc002b9fd40 pc=0x4777ae +runtime.chanrecv(0xc001be5d40, 0xc002b9fe60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002b9fdd8 sp=0xc002b9fd60 pc=0x441325 +runtime.chanrecv1(0xc0026a5208?, 0x40a16a0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002b9fe00 sp=0xc002b9fdd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002b9ffe0 sp=0xc002b9fe00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002b9ffe8 sp=0xc002b9ffe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1376 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1391 [select]: +runtime.gopark(0xc0024c9790?, 0x2?, 0x18?, 0x96?, 0xc0024c9744?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024c95c0 sp=0xc0024c95a0 pc=0x4777ae +runtime.selectgo(0xc0024c9790, 0xc0024c9740, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0024c9710 sp=0xc0024c95c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c6ac60) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc0024c97c0 sp=0xc0024c9710 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc0026a8820, {0x43c4ea0?, 0xc0021e6240}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0024c9a20 sp=0xc0024c97c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001d56894?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc0024c9a80 sp=0xc0024c9a20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001d567e0, 0xc00234ac38, 0xc00234ac08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc0024c9b28 sp=0xc0024c9a80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d567e0, {0x43c4ea0, 0xc0021e6240}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc0024c9c68 sp=0xc0024c9b28 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc001e41f00, {0x43c4ea0, 0xc0021e6240}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc0024c9e30 sp=0xc0024c9c68 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCStdioStreamStdioClient).Recv(0xc002577000) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:160 +0x66 fp=0xc0024c9e70 sp=0xc0024c9e30 pc=0x2103f26 +github.com/hashicorp/go-plugin.(*grpcStdioClient).Run(0xc001d86f80, {0x5b0ee60, 0x779c980}, {0x5b0eb80, 0xc00007c038}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:135 +0x11c fp=0xc0024c9f80 sp=0xc0024c9e70 pc=0x215c15c +github.com/hashicorp/go-plugin.newGRPCClient.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x6c fp=0xc0024c9fe0 sp=0xc0024c9f80 pc=0x21598ec +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024c9fe8 sp=0xc0024c9fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 1361 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x84f + +goroutine 1392 [select]: +runtime.gopark(0xc00234afb0?, 0x2?, 0xb4?, 0x1b?, 0xc00234af74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00234ade8 sp=0xc00234adc8 pc=0x4777ae +runtime.selectgo(0xc00234afb0, 0xc00234af70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00234af38 sp=0xc00234ade8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc00234afe0 sp=0xc00234af38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00234afe8 sp=0xc00234afe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1361 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1393 [select]: +runtime.gopark(0xc0029b7fb0?, 0x2?, 0xb0?, 0x7d?, 0xc0029b7ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b7d58 sp=0xc0029b7d38 pc=0x4777ae +runtime.selectgo(0xc0029b7fb0, 0xc0029b7ee0, 0x5655434a55573374?, 0x0, 0x6a55575255575268?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029b7ea8 sp=0xc0029b7d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0029b7fe0 sp=0xc0029b7ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b7fe8 sp=0xc0029b7fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1410 [chan receive]: +runtime.gopark(0xc0014f90c0?, 0xc001822eb8?, 0xbb?, 0xe2?, 0xc0014f90a0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001822d60 sp=0xc001822d40 pc=0x4777ae +runtime.chanrecv(0xc0022fb500, 0xc001822e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001822dd8 sp=0xc001822d60 pc=0x441325 +runtime.chanrecv1(0xc00146bf60?, 0x10100c001423608?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc001822e00 sp=0xc001822dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc001822fe0 sp=0xc001822e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001822fe8 sp=0xc001822fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1411 [select]: +runtime.gopark(0xc002608fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002608f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002608de8 sp=0xc002608dc8 pc=0x4777ae +runtime.selectgo(0xc002608fb0, 0xc002608f70, 0xc001d9d320?, 0x0, 0xc001d9d380?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002608f38 sp=0xc002608de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002608fe0 sp=0xc002608f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002608fe8 sp=0xc002608fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1294 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1412 [select]: +runtime.gopark(0xc002609fb0?, 0x2?, 0xb0?, 0x9d?, 0xc002609ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002609d58 sp=0xc002609d38 pc=0x4777ae +runtime.selectgo(0xc002609fb0, 0xc002609ee0, 0x203a737365726464?, 0x0, 0xa31313635343a74?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002609ea8 sp=0xc002609d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002609fe0 sp=0xc002609ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002609fe8 sp=0xc002609fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1294 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1413 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002162d60 sp=0xc002162d40 pc=0x4777ae +runtime.chanrecv(0xc0022fb740, 0xc002162e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002162dd8 sp=0xc002162d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002162e00 sp=0xc002162dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002162fe0 sp=0xc002162e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002162fe8 sp=0xc002162fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1294 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1414 [select]: +runtime.gopark(0xc00260afa0?, 0x3?, 0x0?, 0x0?, 0xc00260af3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00260ada8 sp=0xc00260ad88 pc=0x4777ae +runtime.selectgo(0xc00260afa0, 0xc00260af34, 0xc00260af40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00260aef8 sp=0xc00260ada8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:184 +0x1b3 fp=0xc00260afe0 sp=0xc00260aef8 pc=0x21557f3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00260afe8 sp=0xc00260afe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream in goroutine 1294 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:182 +0x24a + +goroutine 1415 [select]: +runtime.gopark(0xc002b9b310?, 0x3?, 0xd9?, 0xea?, 0xc002b9b292?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002b9b0f0 sp=0xc002b9b0d0 pc=0x4777ae +runtime.selectgo(0xc002b9b310, 0xc002b9b28c, 0x46216a2?, 0x0, 0x4?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002b9b240 sp=0xc002b9b0f0 pc=0x48916b +github.com/hashicorp/go-plugin.(*grpcStdioServer).StreamStdio(0xc00234fc60, 0x42219c0?, {0x5b4ebb0, 0xc002577440}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:61 +0x1e5 fp=0xc002b9b350 sp=0xc002b9b240 pc=0x215bbc5 +github.com/hashicorp/go-plugin/internal/plugin._GRPCStdio_StreamStdio_Handler({0x409e580?, 0xc00234fc60}, {0x5b4a6f0, 0xc001d871e0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:185 +0xf7 fp=0xc002b9b3b0 sp=0xc002b9b350 pc=0x2104077 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x409e580, 0xc00234fc60}, {0x5b4ab30, 0xc002711ec0}, 0xc000127980?, 0x47efca8) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002b9b400 sp=0xc002b9b3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x409e580, 0xc00234fc60}, {0x5b4ab30, 0xc002711ec0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002b9b490 sp=0xc002b9b400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x409e580, 0xc00234fc60}, {0x5b492f8, 0xc001389590}, 0xc000127980, 0xc0021e6500) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002b9b7d0 sp=0xc002b9b490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x409e580, 0xc00234fc60}, {0x5b492f8, 0xc001389590}, 0x3f252a0?, 0xc002577370?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002b9b848 sp=0xc002b9b7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b320, 0xc0025cd470, 0x770cba0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002b9bdb0 sp=0xc002b9b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b320, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002b9bf28 sp=0xc002b9bdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002b9bfe0 sp=0xc002b9bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002b9bfe8 sp=0xc002b9bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1387 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1416 [select]: +runtime.gopark(0xc0025e6890?, 0x2?, 0x98?, 0x67?, 0xc0025e683c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0025e66a8 sp=0xc0025e6688 pc=0x4777ae +runtime.selectgo(0xc0025e6890, 0xc0025e6838, 0x7f429a489a00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0025e67f8 sp=0xc0025e66a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0028462d0, {0xc002711fc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0025e68c0 sp=0xc0025e67f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0028462d0, {0xc002711fc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0025e6958 sp=0xc0025e68c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002711b60, {0xc002711fc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0025e69a0 sp=0xc0025e6958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002711b60}, {0xc002711fc0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0025e69e8 sp=0xc0025e69a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c6b560, {0xc002711fc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0025e6a58 sp=0xc0025e69e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002711fb0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0025e6b28 sp=0xc0025e6a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0021e6e80?, 0xc001c6b560, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0025e6c18 sp=0xc0025e6b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002711fb0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc002846780}, 0xc0025ccf60?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0025e6cd0 sp=0xc0025e6c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001389680, {0x43dcf60?, 0xc002846780}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0025e6f08 sp=0xc0025e6cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002848120, {0x43dcf60, 0xc002846780}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0025e6ff0 sp=0xc0025e6f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001d87240, {0x43dcf60, 0xc002846780}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0025e7028 sp=0xc0025e6ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc002577550) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc0025e7068 sp=0xc0025e7028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc002577550?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0025e70a8 sp=0xc0025e7068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc002577560) + :1 +0x46 fp=0xc0025e70e0 sp=0xc0025e70a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0025cd0e0, {0x5b51368, 0xc002577560}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc0025e7320 sp=0xc0025e70e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00234fbc0, {0x5b51038, 0xc002577550}) + :1 +0xaf fp=0xc0025e7370 sp=0xc0025e7320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00234fbc0}, {0x5b4a6f0?, 0xc001d87240}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0025e73b0 sp=0xc0025e7370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc002848120}, 0xc000127b48?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0025e7400 sp=0xc0025e73b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc002848120}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0025e7490 sp=0xc0025e7400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc001389680}, 0xc000127b48, 0xc0021e6700) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0025e77d0 sp=0xc0025e7490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc001389680}, 0x3f252a0?, 0xc002577480?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0025e7848 sp=0xc0025e77d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b560, 0xc0025cd170, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0025e7db0 sp=0xc0025e7848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b560, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0025e7f28 sp=0xc0025e7db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0025e7fe0 sp=0xc0025e7f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0025e7fe8 sp=0xc0025e7fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1387 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1417 [select]: +runtime.gopark(0xc0027c2a68?, 0x2?, 0x40?, 0xa5?, 0xc0027c2a14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027c2880 sp=0xc0027c2860 pc=0x4777ae +runtime.selectgo(0xc0027c2a68, 0xc0027c2a10, 0x7f429a489700?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0027c29d0 sp=0xc0027c2880 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0028463c0, {0xc0028482b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0027c2a98 sp=0xc0027c29d0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0028463c0, {0xc0028482b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0027c2b30 sp=0xc0027c2a98 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002711ce0, {0xc0028482b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0027c2b78 sp=0xc0027c2b30 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002711ce0}, {0xc0028482b0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0027c2bc0 sp=0xc0027c2b78 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c6b680, {0xc0028482b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0027c2c30 sp=0xc0027c2bc0 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0028482a0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0027c2d00 sp=0xc0027c2c30 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc0027c1570?, 0xc001c6b680, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0027c2df0 sp=0xc0027c2d00 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0028482a0, {0x7f424347ae80, 0x779c980}, 0xc001389680?, {0x0, 0x0}, {0x43dd860, 0xc002846870}, 0x12b01b3?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0027c2ea8 sp=0xc0027c2df0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001389770, {0x43dd860?, 0xc002846870}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0027c30e0 sp=0xc0027c2ea8 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0028483f0, {0x43dd860, 0xc002846870}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0027c31c8 sp=0xc0027c30e0 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc001d872c0, {0x43dd860, 0xc002846870}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0027c3200 sp=0xc0027c31c8 pc=0x1fd94b2 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamServer).Recv(0xc002577690) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:167 +0x66 fp=0xc0027c3240 sp=0xc0027c3200 pc=0x2102886 +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream(0xc0025cd230, {0x5b512b8, 0xc002577690}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:87 +0x218 fp=0xc0027c3370 sp=0xc0027c3240 pc=0x2154718 +github.com/hashicorp/go-plugin/internal/plugin._GRPCBroker_StartStream_Handler({0x41cb860?, 0xc0025cd230}, {0x5b4a6f0?, 0xc001d872c0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:148 +0xb6 fp=0xc0027c33b0 sp=0xc0027c3370 pc=0x2102716 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41cb860, 0xc0025cd230}, {0x5b4ab30, 0xc0028483f0}, 0xc000127f08?, 0x47efc98) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0027c3400 sp=0xc0027c33b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41cb860, 0xc0025cd230}, {0x5b4ab30, 0xc0028483f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0027c3490 sp=0xc0027c3400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41cb860, 0xc0025cd230}, {0x5b492f8, 0xc001389770}, 0xc000127f08, 0xc0021e6ec0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0027c37d0 sp=0xc0027c3490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41cb860, 0xc0025cd230}, {0x5b492f8, 0xc001389770}, 0x3f252a0?, 0xc0025775c0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0027c3848 sp=0xc0027c37d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b680, 0xc0025cd2c0, 0x770cb80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0027c3db0 sp=0xc0027c3848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b680, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0027c3f28 sp=0xc0027c3db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0027c3fe0 sp=0xc0027c3f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0027c3fe8 sp=0xc0027c3fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1387 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1418 [select]: +runtime.gopark(0xc0029b3fa0?, 0x3?, 0x0?, 0x4e?, 0xc0029b3f3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b3da8 sp=0xc0029b3d88 pc=0x4777ae +runtime.selectgo(0xc0029b3fa0, 0xc0029b3f34, 0xc0029b3f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029b3ef8 sp=0xc0029b3da8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:73 +0x1ad fp=0xc0029b3fe0 sp=0xc0029b3ef8 pc=0x2154aad +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b3fe8 sp=0xc0029b3fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream in goroutine 1417 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:71 +0x1d1 + +goroutine 1432 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00260d9e0 sp=0xc00260d9c0 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00260da18 sp=0xc00260d9e0 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c45c0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00260da38 sp=0xc00260da18 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc002c192a0, 0x448801?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00260da80 sp=0xc00260da38 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc002c19280) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc00260db70 sp=0xc00260da80 pc=0x513f65 +net.(*netFD).accept(0xc002c19280) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc00260dc68 sp=0xc00260db70 pc=0x8967be +net.(*TCPListener).accept(0xc002bf8b80) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc00260dcb0 sp=0xc00260dc68 pc=0x8c383e +net.(*TCPListener).Accept(0xc002bf8b80) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc00260dd20 sp=0xc00260dcb0 pc=0x8c1ba5 +net/http.(*onceCloseListener).Accept(0xc002781ec0) + :1 +0x44 fp=0xc00260dd58 sp=0xc00260dd20 pc=0xab5544 +net/http.(*Server).Serve(0xc00278a1e0, {0x5b2a6d0, 0xc002bf8b80}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc00260df50 sp=0xc00260dd58 pc=0xa7859f +github.com/rancher/opni/pkg/util.ServeHandler.func1() + /home/ec2-user/opni/pkg/util/servers.go:19 +0x47 fp=0xc00260dfa0 sp=0xc00260df50 pc=0x176aa67 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc00260dfe0 sp=0xc00260dfa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00260dfe8 sp=0xc00260dfe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1477 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1426 [select]: +runtime.gopark(0xc00260cfb0?, 0x2?, 0xb4?, 0x1b?, 0xc00260cf74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00260cde8 sp=0xc00260cdc8 pc=0x4777ae +runtime.selectgo(0xc00260cfb0, 0xc00260cf70, 0xc00235c9c0?, 0x0, 0xc00235ca20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00260cf38 sp=0xc00260cde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc00260cfe0 sp=0xc00260cf38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00260cfe8 sp=0xc00260cfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1361 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1427 [select]: +runtime.gopark(0xc002106fb0?, 0x2?, 0xb0?, 0x6d?, 0xc002106ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002106d58 sp=0xc002106d38 pc=0x4777ae +runtime.selectgo(0xc002106fb0, 0xc002106ee0, 0xc00073add1?, 0x0, 0x5b058a8?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002106ea8 sp=0xc002106d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002106fe0 sp=0xc002106ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002106fe8 sp=0xc002106fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1428 [chan receive]: +runtime.gopark(0xc002405da8?, 0x1103325?, 0x0?, 0xa7?, 0xc0025253a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002405d60 sp=0xc002405d40 pc=0x4777ae +runtime.chanrecv(0xc00235cea0, 0xc002405e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002405dd8 sp=0xc002405d60 pc=0x441325 +runtime.chanrecv1(0xc0026525d0?, 0xc000f1f560?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002405e00 sp=0xc002405dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002405fe0 sp=0xc002405e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002405fe8 sp=0xc002405fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1435 [select]: +runtime.gopark(0xc002d87fb0?, 0x2?, 0x0?, 0x0?, 0xc002d87f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d87da8 sp=0xc002d87d88 pc=0x4777ae +runtime.selectgo(0xc002d87fb0, 0xc002d87f28, 0x0?, 0x0, 0x4ab492?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d87ef8 sp=0xc002d87da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002d87fe0 sp=0xc002d87ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d87fe8 sp=0xc002d87fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1419 [select]: +runtime.gopark(0xc002774890?, 0x2?, 0x4e?, 0x1a?, 0xc00277483c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027746a8 sp=0xc002774688 pc=0x4777ae +runtime.selectgo(0xc002774890, 0xc002774838, 0x7f429a488200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0027747f8 sp=0xc0027746a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002846960, {0xc002780fd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0027748c0 sp=0xc0027747f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002846960, {0xc002780fd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002774958 sp=0xc0027748c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0028485d0, {0xc002780fd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0027749a0 sp=0xc002774958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0028485d0}, {0xc002780fd0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0027749e8 sp=0xc0027749a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c6b7a0, {0xc002780fd0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002774a58 sp=0xc0027749e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002780fc0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002774b28 sp=0xc002774a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc00223ad00?, 0xc001c6b7a0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002774c18 sp=0xc002774b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002780fc0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0025fb450}, 0xc0025ccf60?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002774cd0 sp=0xc002774c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc00278a000, {0x43dcf60?, 0xc0025fb450}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002774f08 sp=0xc002774cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002781110, {0x43dcf60, 0xc0025fb450}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002774ff0 sp=0xc002774f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002181800, {0x43dcf60, 0xc0025fb450}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002775028 sp=0xc002774ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc002856c30) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc002775068 sp=0xc002775028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc002856c30?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc0027750a8 sp=0xc002775068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc002856c40) + :1 +0x46 fp=0xc0027750e0 sp=0xc0027750a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0025cd0e0, {0x5b51368, 0xc002856c40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc002775320 sp=0xc0027750e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00234fbc0, {0x5b51038, 0xc002856c30}) + :1 +0xaf fp=0xc002775370 sp=0xc002775320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00234fbc0}, {0x5b4a6f0?, 0xc002181800}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc0027753b0 sp=0xc002775370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc002781110}, 0xc00149eac8?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002775400 sp=0xc0027753b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc002781110}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002775490 sp=0xc002775400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc00278a000}, 0xc00149eac8, 0xc00223a600) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0027757d0 sp=0xc002775490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc00278a000}, 0x3f252a0?, 0xc002856b60?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002775848 sp=0xc0027757d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b7a0, 0xc0025cd170, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002775db0 sp=0xc002775848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc001c6b7a0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002775f28 sp=0xc002775db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002775fe0 sp=0xc002775f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002775fe8 sp=0xc002775fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1387 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1448 [chan receive]: +runtime.gopark(0xc0007defc0?, 0xc0007df0a0?, 0xf0?, 0xf1?, 0xc0007df2d0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b0d60 sp=0xc0029b0d40 pc=0x4777ae +runtime.chanrecv(0xc001f03a40, 0xc0029b0e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0029b0dd8 sp=0xc0029b0d60 pc=0x441325 +runtime.chanrecv1(0xc000ee7320?, 0xc00256ec80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0029b0e00 sp=0xc0029b0dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0029b0fe0 sp=0xc0029b0e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b0fe8 sp=0xc0029b0fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1361 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1455 [select]: +runtime.gopark(0xc002102fb0?, 0x2?, 0x0?, 0x4e?, 0xc002102f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002102da8 sp=0xc002102d88 pc=0x4777ae +runtime.selectgo(0xc002102fb0, 0xc002102f28, 0x4b0571?, 0x0, 0xc002102fd0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002102ef8 sp=0xc002102da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002102fe0 sp=0xc002102ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002102fe8 sp=0xc002102fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1454 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1450 [select]: +runtime.gopark(0xc002ba6890?, 0x2?, 0x98?, 0x67?, 0xc002ba683c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ba66a8 sp=0xc002ba6688 pc=0x4777ae +runtime.selectgo(0xc002ba6890, 0xc002ba6838, 0x7f429a486a00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ba67f8 sp=0xc002ba66a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002658fa0, {0xc0026538a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002ba68c0 sp=0xc002ba67f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002658fa0, {0xc0026538a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002ba6958 sp=0xc002ba68c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002653680, {0xc0026538a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002ba69a0 sp=0xc002ba6958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002653680}, {0xc0026538a0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002ba69e8 sp=0xc002ba69a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0018f58c0, {0xc0026538a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002ba6a58 sp=0xc002ba69e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002653890, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002ba6b28 sp=0xc002ba6a58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc001d7c780?, 0xc0018f58c0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002ba6c18 sp=0xc002ba6b28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002653890, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc0026593b0}, 0xc0025ccf60?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002ba6cd0 sp=0xc002ba6c18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000b9c1e0, {0x43dcf60?, 0xc0026593b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002ba6f08 sp=0xc002ba6cd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0026539e0, {0x43dcf60, 0xc0026593b0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002ba6ff0 sp=0xc002ba6f08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002488160, {0x43dcf60, 0xc0026593b0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002ba7028 sp=0xc002ba6ff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc00271e780) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc002ba7068 sp=0xc002ba7028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc00271e780?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc002ba70a8 sp=0xc002ba7068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc00271e790) + :1 +0x46 fp=0xc002ba70e0 sp=0xc002ba70a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0025cd0e0, {0x5b51368, 0xc00271e790}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc002ba7320 sp=0xc002ba70e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc00234fbc0, {0x5b51038, 0xc00271e780}) + :1 +0xaf fp=0xc002ba7370 sp=0xc002ba7320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc00234fbc0}, {0x5b4a6f0?, 0xc002488160}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc002ba73b0 sp=0xc002ba7370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc0026539e0}, 0xc0005817d0?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002ba7400 sp=0xc002ba73b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc00234fbc0}, {0x5b4ab30, 0xc0026539e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002ba7490 sp=0xc002ba7400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc000b9c1e0}, 0xc0005817d0, 0xc002171f80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002ba77d0 sp=0xc002ba7490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc00234fbc0}, {0x5b492f8, 0xc000b9c1e0}, 0x3f252a0?, 0xc00271e6b0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002ba7848 sp=0xc002ba77d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc0018f58c0, 0xc0025cd170, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002ba7db0 sp=0xc002ba7848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0018534a0, {0x5b54e00, 0xc0024e7860}, 0xc0018f58c0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002ba7f28 sp=0xc002ba7db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002ba7fe0 sp=0xc002ba7f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ba7fe8 sp=0xc002ba7fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1387 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1456 [select]: +runtime.gopark(0xc002c84f90?, 0x2?, 0xff?, 0xff?, 0xc002c84eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c84d28 sp=0xc002c84d08 pc=0x4777ae +runtime.selectgo(0xc002c84f90, 0xc002c84ea8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c84e78 sp=0xc002c84d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc0015ec240) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002c84fc0 sp=0xc002c84e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002c84fe0 sp=0xc002c84fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c84fe8 sp=0xc002c84fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1454 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1452 [IO wait]: +runtime.gopark(0x0?, 0x43e546?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002107930 sp=0xc002107910 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x513f65?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002107968 sp=0xc002107930 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c46b8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002107988 sp=0xc002107968 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00182d720, 0x448801?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc0021079d0 sp=0xc002107988 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc00182d700) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:611 +0x405 fp=0xc002107ac0 sp=0xc0021079d0 pc=0x513f65 +net.(*netFD).accept(0xc00182d700) + /home/ec2-user/sdk/go1.21.0/src/net/fd_unix.go:172 +0x3e fp=0xc002107bb8 sp=0xc002107ac0 pc=0x8967be +net.(*TCPListener).accept(0xc002488200) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock_posix.go:152 +0x3e fp=0xc002107c00 sp=0xc002107bb8 pc=0x8c383e +net.(*TCPListener).Accept(0xc002488200) + /home/ec2-user/sdk/go1.21.0/src/net/tcpsock.go:315 +0x65 fp=0xc002107c70 sp=0xc002107c00 pc=0x8c1ba5 +net/http.(*onceCloseListener).Accept(0xc002781380) + :1 +0x44 fp=0xc002107ca8 sp=0xc002107c70 pc=0xab5544 +net/http.(*Server).Serve(0xc00278a0f0, {0x5b2a6d0, 0xc002488200}) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:3056 +0x57f fp=0xc002107ea0 sp=0xc002107ca8 pc=0xa7859f +net/http.Serve(...) + /home/ec2-user/sdk/go1.21.0/src/net/http/server.go:2595 +github.com/gin-gonic/gin.(*Engine).RunListener(0xc0025eeea0, {0x5b2a6d0, 0xc002488200}) + /home/ec2-user/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/gin.go:565 +0x370 fp=0xc002107f90 sp=0xc002107ea0 pc=0x171e870 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure.func1() + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:72 +0x5c fp=0xc002107fe0 sp=0xc002107f90 pc=0x262a57c +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002107fe8 sp=0xc002107fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/plugins/apis/apiextensions/http.(*httpApiExtensionPlugin).Configure in goroutine 1451 + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/http/plugin.go:71 +0x26d + +goroutine 1421 [select]: +runtime.gopark(0xc002bf3e70?, 0x2?, 0x0?, 0x0?, 0xc002bf3df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002bf3c58 sp=0xc002bf3c38 pc=0x4777ae +runtime.selectgo(0xc002bf3e70, 0xc002bf3df0, 0xc002bf3e00?, 0x0, 0xc00125fc00?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002bf3da8 sp=0xc002bf3c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002846a50, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002bf3ea0 sp=0xc002bf3da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002715570) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002bf3f58 sp=0xc002bf3ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc002bf3fe0 sp=0xc002bf3f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002bf3fe8 sp=0xc002bf3fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1372 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1396 [select]: +runtime.gopark(0xc002bebe48?, 0x2?, 0x0?, 0x0?, 0xc002bebdcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002bebc30 sp=0xc002bebc10 pc=0x4777ae +runtime.selectgo(0xc002bebe48, 0xc002bebdc8, 0xc002bebdd8?, 0x0, 0xc00133b500?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002bebd80 sp=0xc002bebc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0026a51d0, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002bebe78 sp=0xc002bebd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0026ae700) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002bebf30 sp=0xc002bebe78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002bebfe0 sp=0xc002bebf30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002bebfe8 sp=0xc002bebfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1395 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1397 [select]: +runtime.gopark(0xc00260ef70?, 0x4?, 0x40?, 0x0?, 0xc00260edd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00260ec00 sp=0xc00260ebe0 pc=0x4777ae +runtime.selectgo(0xc00260ef70, 0xc00260edd0, 0xc00260ef08?, 0x0, 0x5b50310?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00260ed50 sp=0xc00260ec00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002515860) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc00260efc0 sp=0xc00260ed50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc00260efe0 sp=0xc00260efc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00260efe8 sp=0xc00260efe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1395 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1398 [IO wait]: +runtime.gopark(0xc001730f88?, 0x48eb51?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001730f40 sp=0xc001730f20 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc001730f78 sp=0xc001730f40 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c47b0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc001730f98 sp=0xc001730f78 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc00257daa0, 0xc002c6ea00?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc001730fe0 sp=0xc001730f98 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00257da80, {0xc002c6ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc0017310d0 sp=0xc001730fe0 pc=0x50e5a5 +net.(*netFD).Read(0xc00257da80, {0xc002c6ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc001731130 sp=0xc0017310d0 pc=0x89342b +net.(*conn).Read(0xc001d6f078, {0xc002c6ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc0017311b8 sp=0xc001731130 pc=0x8b176d +net.(*UnixConn).Read(0xc001d6f078, {0xc002c6ea00, 0x500, 0x500}) + :1 +0x54 fp=0xc001731200 sp=0xc0017311b8 pc=0x8d8df4 +crypto/tls.(*atLeastReader).Read(0xc0030c4528, {0xc002c6ea00, 0x500, 0x500}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:805 +0x7e fp=0xc001731260 sp=0xc001731200 pc=0x945fde +bytes.(*Buffer).ReadFrom(0xc00133b7a8, {0x5b0f6c0, 0xc0030c4528}) + /home/ec2-user/sdk/go1.21.0/src/bytes/buffer.go:211 +0x110 fp=0xc0017312c8 sp=0xc001731260 pc=0x5bf530 +crypto/tls.(*Conn).readFromUntil(0xc00133b500, {0x5b0f840?, 0xc001d6f078}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:827 +0x1ea fp=0xc001731338 sp=0xc0017312c8 pc=0x9463aa +crypto/tls.(*Conn).readRecordOrCCS(0xc00133b500, 0x0) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:625 +0x606 fp=0xc001731968 sp=0xc001731338 pc=0x942046 +crypto/tls.(*Conn).readRecord(...) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:587 +crypto/tls.(*Conn).Read(0xc00133b500, {0xc002bcc000, 0x8000, 0x4b0571?}) + /home/ec2-user/sdk/go1.21.0/src/crypto/tls/conn.go:1369 +0x270 fp=0xc001731a50 sp=0xc001731968 pc=0x94bcb0 +google.golang.org/grpc/internal/credentials.(*syscallConn).Read(0xc0026e0640, {0xc002bcc000, 0x8000, 0x8000}) + :1 +0x6d fp=0xc001731a98 sp=0xc001731a50 pc=0x10969ed +bufio.(*Reader).Read(0xc001de70e0, {0xc002686660, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc001731b58 sp=0xc001731a98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc001de70e0}, {0xc002686660, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001731ba0 sp=0xc001731b58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc002686660, 0x9, 0x9}, {0x5b0f0a0, 0xc001de70e0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc001731c40 sp=0xc001731ba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc002686620) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc001731d78 sp=0xc001731c40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002515860, 0x2020200a3936202d?, 0x200a313131202d20?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc001731ec8 sp=0xc001731d78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc001853680, {0x5b54e00?, 0xc002515860}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc001731f80 sp=0xc001731ec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc001731fe0 sp=0xc001731f80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001731fe8 sp=0xc001731fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1395 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1402 [select]: +runtime.gopark(0xc002ba0fa0?, 0x3?, 0x0?, 0x0?, 0xc002ba0f3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ba0da8 sp=0xc002ba0d88 pc=0x4777ae +runtime.selectgo(0xc002ba0fa0, 0xc002ba0f34, 0xc002ba0f40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ba0ef8 sp=0xc002ba0da8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:184 +0x1b3 fp=0xc002ba0fe0 sp=0xc002ba0ef8 pc=0x21557f3 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ba0fe8 sp=0xc002ba0fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerClientImpl).StartStream in goroutine 1376 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:182 +0x24a + +goroutine 1403 [select]: +runtime.gopark(0xc002ba1fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002ba1f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ba1de8 sp=0xc002ba1dc8 pc=0x4777ae +runtime.selectgo(0xc002ba1fb0, 0xc002ba1f70, 0x21557a5?, 0x0, 0xc0026882c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ba1f38 sp=0xc002ba1de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002ba1fe0 sp=0xc002ba1f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ba1fe8 sp=0xc002ba1fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1367 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1404 [select]: +runtime.gopark(0xc002c0ffb0?, 0x2?, 0xb0?, 0xfd?, 0xc002c0fee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c0fd58 sp=0xc002c0fd38 pc=0x4777ae +runtime.selectgo(0xc002c0ffb0, 0xc002c0fee0, 0x1128d01?, 0x0, 0x1102359?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c0fea8 sp=0xc002c0fd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002c0ffe0 sp=0xc002c0fea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c0ffe8 sp=0xc002c0ffe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1405 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ba2d60 sp=0xc002ba2d40 pc=0x4777ae +runtime.chanrecv(0xc002bf6120, 0xc002ba2e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002ba2dd8 sp=0xc002ba2d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002ba2e00 sp=0xc002ba2dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002ba2fe0 sp=0xc002ba2e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ba2fe8 sp=0xc002ba2fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1406 [select]: +runtime.gopark(0xc0027c7790?, 0x2?, 0x18?, 0x76?, 0xc0027c7744?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027c75c0 sp=0xc0027c75a0 pc=0x4777ae +runtime.selectgo(0xc0027c7790, 0xc0027c7740, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0027c7710 sp=0xc0027c75c0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001d570e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x114 fp=0xc0027c77c0 sp=0xc0027c7710 pc=0x113c5d4 +google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002782c30, {0x43c4ea0?, 0xc002452140}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0x14f fp=0xc0027c7a20 sp=0xc0027c77c0 pc=0x119216f +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0019cabf4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc0027c7a80 sp=0xc0027c7a20 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0019cab40, 0xc002ba1c38, 0xc002ba1c08) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc0027c7b28 sp=0xc0027c7a80 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019cab40, {0x43c4ea0, 0xc002452140}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc0027c7c68 sp=0xc0027c7b28 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc001ff1e80, {0x43c4ea0, 0xc002452140}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc0027c7e30 sp=0xc0027c7c68 pc=0x12ac5b9 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCStdioStreamStdioClient).Recv(0xc002688340) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:160 +0x66 fp=0xc0027c7e70 sp=0xc0027c7e30 pc=0x2103f26 +github.com/hashicorp/go-plugin.(*grpcStdioClient).Run(0xc0026e0a60, {0x5b0ee60, 0x779c980}, {0x5b0eb80, 0xc00007c038}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:135 +0x11c fp=0xc0027c7f80 sp=0xc0027c7e70 pc=0x215c15c +github.com/hashicorp/go-plugin.newGRPCClient.func3() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x6c fp=0xc0027c7fe0 sp=0xc0027c7f80 pc=0x21598ec +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0027c7fe8 sp=0xc0027c7fe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.newGRPCClient in goroutine 1367 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_client.go:75 +0x84f + +goroutine 1407 [select]: +runtime.gopark(0xc002b9cfb0?, 0x2?, 0xb4?, 0x1b?, 0xc002b9cf74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002b9cde8 sp=0xc002b9cdc8 pc=0x4777ae +runtime.selectgo(0xc002b9cfb0, 0xc002b9cf70, 0xa3939202d202020?, 0x0, 0x31202d202020200a?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002b9cf38 sp=0xc002b9cde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002b9cfe0 sp=0xc002b9cf38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002b9cfe8 sp=0xc002b9cfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1367 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1408 [select]: +runtime.gopark(0xc002b9dfb0?, 0x2?, 0xb0?, 0xdd?, 0xc002b9dee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002b9dd58 sp=0xc002b9dd38 pc=0x4777ae +runtime.selectgo(0xc002b9dfb0, 0xc002b9dee0, 0x20200a3038202d20?, 0x0, 0xa3038202d202020?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002b9dea8 sp=0xc002b9dd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002b9dfe0 sp=0xc002b9dea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002b9dfe8 sp=0xc002b9dfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1409 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002b9ed60 sp=0xc002b9ed40 pc=0x4777ae +runtime.chanrecv(0xc002bf63c0, 0xc002b9ee60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002b9edd8 sp=0xc002b9ed60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002b9ee00 sp=0xc002b9edd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002b9efe0 sp=0xc002b9ee00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002b9efe8 sp=0xc002b9efe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1458 [select]: +runtime.gopark(0xc0024dea68?, 0x2?, 0xfe?, 0x1c?, 0xc0024dea14?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0024de880 sp=0xc0024de860 pc=0x4777ae +runtime.selectgo(0xc0024dea68, 0xc0024dea10, 0x7f429a48c700?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0024de9d0 sp=0xc0024de880 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0026a5900, {0xc0026ab870, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0024dea98 sp=0xc0024de9d0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0026a5900, {0xc0026ab870, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0024deb30 sp=0xc0024dea98 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0026ab4a0, {0xc0026ab870, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0024deb78 sp=0xc0024deb30 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0026ab4a0}, {0xc0026ab870, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0024debc0 sp=0xc0024deb78 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d57560, {0xc0026ab870, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0024dec30 sp=0xc0024debc0 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0026ab860, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0024ded00 sp=0xc0024dec30 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x98b2c5?, 0xc001d57560, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0024dedf0 sp=0xc0024ded00 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0026ab860, {0x7f424347ae80, 0x779c980}, 0x6aa1293eacb061af?, {0x0, 0x0}, {0x43dd860, 0xc0026a5bd0}, 0x2a1b853c16d73cc7?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0024deea8 sp=0xc0024dedf0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0016704b0, {0x43dd860?, 0xc0026a5bd0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0024df0e0 sp=0xc0024deea8 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0026ab9b0, {0x43dd860, 0xc0026a5bd0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0024df1c8 sp=0xc0024df0e0 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002bf80e0, {0x43dd860, 0xc0026a5bd0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0024df200 sp=0xc0024df1c8 pc=0x1fd94b2 +github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamServer).Recv(0xc002688760) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:167 +0x66 fp=0xc0024df240 sp=0xc0024df200 pc=0x2102886 +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream(0xc0025cdda0, {0x5b512b8, 0xc002688760}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:87 +0x218 fp=0xc0024df370 sp=0xc0024df240 pc=0x2154718 +github.com/hashicorp/go-plugin/internal/plugin._GRPCBroker_StartStream_Handler({0x41cb860?, 0xc0025cdda0}, {0x5b4a6f0?, 0xc002bf80e0}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_broker.pb.go:148 +0xb6 fp=0xc0024df3b0 sp=0xc0024df370 pc=0x2102716 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41cb860, 0xc0025cdda0}, {0x5b4ab30, 0xc0026ab9b0}, 0xc000dc2558?, 0x47efc98) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0024df400 sp=0xc0024df3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41cb860, 0xc0025cdda0}, {0x5b4ab30, 0xc0026ab9b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0024df490 sp=0xc0024df400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41cb860, 0xc0025cdda0}, {0x5b492f8, 0xc0016704b0}, 0xc000dc2558, 0xc002452480) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0024df7d0 sp=0xc0024df490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41cb860, 0xc0025cdda0}, {0x5b492f8, 0xc0016704b0}, 0x3f252a0?, 0xc002688690?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0024df848 sp=0xc0024df7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d57560, 0xc0025cde30, 0x770cb80, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0024dfdb0 sp=0xc0024df848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d57560, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0024dff28 sp=0xc0024dfdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0024dffe0 sp=0xc0024dff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0024dffe8 sp=0xc0024dffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1398 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1459 [select]: +runtime.gopark(0xc002c17310?, 0x3?, 0xd9?, 0xea?, 0xc002c17292?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c170f0 sp=0xc002c170d0 pc=0x4777ae +runtime.selectgo(0xc002c17310, 0xc002c1728c, 0x46216a2?, 0x0, 0x4?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c17240 sp=0xc002c170f0 pc=0x48916b +github.com/hashicorp/go-plugin.(*grpcStdioServer).StreamStdio(0xc0026e3900, 0x42219c0?, {0x5b4ebb0, 0xc002688840}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_stdio.go:61 +0x1e5 fp=0xc002c17350 sp=0xc002c17240 pc=0x215bbc5 +github.com/hashicorp/go-plugin/internal/plugin._GRPCStdio_StreamStdio_Handler({0x409e580?, 0xc0026e3900}, {0x5b4a6f0, 0xc002bf8140}) + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/internal/plugin/grpc_stdio.pb.go:185 +0xf7 fp=0xc002c173b0 sp=0xc002c17350 pc=0x2104077 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x409e580, 0xc0026e3900}, {0x5b4ab30, 0xc0026abbf0}, 0xc000dc25a0?, 0x47efca8) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002c17400 sp=0xc002c173b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x409e580, 0xc0026e3900}, {0x5b4ab30, 0xc0026abbf0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002c17490 sp=0xc002c17400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x409e580, 0xc0026e3900}, {0x5b492f8, 0xc0016705a0}, 0xc000dc25a0, 0xc002452600) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002c177d0 sp=0xc002c17490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x409e580, 0xc0026e3900}, {0x5b492f8, 0xc0016705a0}, 0x3f252a0?, 0xc002688770?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002c17848 sp=0xc002c177d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d57680, 0xc002780000, 0x770cba0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002c17db0 sp=0xc002c17848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d57680, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002c17f28 sp=0xc002c17db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002c17fe0 sp=0xc002c17f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c17fe8 sp=0xc002c17fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1398 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1460 [select]: +runtime.gopark(0xc00218c890?, 0x2?, 0x98?, 0xc7?, 0xc00218c83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00218c6a8 sp=0xc00218c688 pc=0x4777ae +runtime.selectgo(0xc00218c890, 0xc00218c838, 0x7f429a48c100?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00218c7f8 sp=0xc00218c6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0026a5ae0, {0xc0026abcf0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc00218c8c0 sp=0xc00218c7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0026a5ae0, {0xc0026abcf0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc00218c958 sp=0xc00218c8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0026ab7a0, {0xc0026abcf0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc00218c9a0 sp=0xc00218c958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0026ab7a0}, {0xc0026abcf0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00218c9e8 sp=0xc00218c9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d577a0, {0xc0026abcf0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc00218ca58 sp=0xc00218c9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0026abce0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc00218cb28 sp=0xc00218ca58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc002452e00?, 0xc001d577a0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc00218cc18 sp=0xc00218cb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0026abce0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc002bfc050}, 0xc0025cdad0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc00218ccd0 sp=0xc00218cc18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001670690, {0x43dcf60?, 0xc002bfc050}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc00218cf08 sp=0xc00218ccd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0026abe30, {0x43dcf60, 0xc002bfc050}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc00218cff0 sp=0xc00218cf08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002bf81a0, {0x43dcf60, 0xc002bfc050}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc00218d028 sp=0xc00218cff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc002688920) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc00218d068 sp=0xc00218d028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc002688920?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc00218d0a8 sp=0xc00218d068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc002688930) + :1 +0x46 fp=0xc00218d0e0 sp=0xc00218d0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0025cdc50, {0x5b51368, 0xc002688930}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc00218d320 sp=0xc00218d0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc0026e2fa0, {0x5b51038, 0xc002688920}) + :1 +0xaf fp=0xc00218d370 sp=0xc00218d320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc0026e2fa0}, {0x5b4a6f0?, 0xc002bf81a0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc00218d3b0 sp=0xc00218d370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc0026e2fa0}, {0x5b4ab30, 0xc0026abe30}, 0xc000dc25e8?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc00218d400 sp=0xc00218d3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc0026e2fa0}, {0x5b4ab30, 0xc0026abe30}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc00218d490 sp=0xc00218d400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc0026e2fa0}, {0x5b492f8, 0xc001670690}, 0xc000dc25e8, 0xc002452780) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc00218d7d0 sp=0xc00218d490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc0026e2fa0}, {0x5b492f8, 0xc001670690}, 0x3f252a0?, 0xc002688850?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc00218d848 sp=0xc00218d7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d577a0, 0xc0025cdce0, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc00218ddb0 sp=0xc00218d848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d577a0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc00218df28 sp=0xc00218ddb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc00218dfe0 sp=0xc00218df28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00218dfe8 sp=0xc00218dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1398 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1461 [select]: +runtime.gopark(0xc002c0bfa0?, 0x3?, 0x0?, 0x4e?, 0xc002c0bf3a?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c0bda8 sp=0xc002c0bd88 pc=0x4777ae +runtime.selectgo(0xc002c0bfa0, 0xc002c0bf34, 0xc002c0bf40?, 0x0, 0x3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c0bef8 sp=0xc002c0bda8 pc=0x48916b +github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream.func1() + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:73 +0x1ad fp=0xc002c0bfe0 sp=0xc002c0bef8 pc=0x2154aad +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c0bfe8 sp=0xc002c0bfe0 pc=0x4ad421 +created by github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream in goroutine 1458 + /home/ec2-user/go/pkg/mod/github.com/alexandre!lamarre/go-plugin@v0.1.1-0.20230417174342-eab684801be5/grpc_broker.go:71 +0x1d1 + +goroutine 1467 [select]: +runtime.gopark(0xc001bce890?, 0x2?, 0x98?, 0xe7?, 0xc001bce83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001bce6a8 sp=0xc001bce688 pc=0x4777ae +runtime.selectgo(0xc001bce890, 0xc001bce838, 0x7f429a48bb00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001bce7f8 sp=0xc001bce6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002bfc5a0, {0xc002bfe580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc001bce8c0 sp=0xc001bce7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002bfc5a0, {0xc002bfe580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc001bce958 sp=0xc001bce8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002bfe4e0, {0xc002bfe580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc001bce9a0 sp=0xc001bce958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002bfe4e0}, {0xc002bfe580, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc001bce9e8 sp=0xc001bce9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d57d40, {0xc002bfe580, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc001bcea58 sp=0xc001bce9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002bfe570, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc001bceb28 sp=0xc001bcea58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc002453b80?, 0xc001d57d40, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc001bcec18 sp=0xc001bceb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002bfe570, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc002bfc960}, 0xc0025cdad0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc001bcecd0 sp=0xc001bcec18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001670780, {0x43dcf60?, 0xc002bfc960}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc001bcef08 sp=0xc001bcecd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002bfe6c0, {0x43dcf60, 0xc002bfc960}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc001bceff0 sp=0xc001bcef08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002bf8380, {0x43dcf60, 0xc002bfc960}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc001bcf028 sp=0xc001bceff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc002688d10) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc001bcf068 sp=0xc001bcf028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc002688d10?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc001bcf0a8 sp=0xc001bcf068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc002688d20) + :1 +0x46 fp=0xc001bcf0e0 sp=0xc001bcf0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0025cdc50, {0x5b51368, 0xc002688d20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc001bcf320 sp=0xc001bcf0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc0026e2fa0, {0x5b51038, 0xc002688d10}) + :1 +0xaf fp=0xc001bcf370 sp=0xc001bcf320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc0026e2fa0}, {0x5b4a6f0?, 0xc002bf8380}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc001bcf3b0 sp=0xc001bcf370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc0026e2fa0}, {0x5b4ab30, 0xc002bfe6c0}, 0xc000dc2c00?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc001bcf400 sp=0xc001bcf3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc0026e2fa0}, {0x5b4ab30, 0xc002bfe6c0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc001bcf490 sp=0xc001bcf400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc0026e2fa0}, {0x5b492f8, 0xc001670780}, 0xc000dc2c00, 0xc002453500) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc001bcf7d0 sp=0xc001bcf490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc0026e2fa0}, {0x5b492f8, 0xc001670780}, 0x3f252a0?, 0xc002688c40?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc001bcf848 sp=0xc001bcf7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d57d40, 0xc0025cdce0, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc001bcfdb0 sp=0xc001bcf848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d57d40, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc001bcff28 sp=0xc001bcfdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc001bcffe0 sp=0xc001bcff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001bcffe8 sp=0xc001bcffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1398 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1463 [select]: +runtime.gopark(0xc002c0dfb0?, 0x2?, 0xb4?, 0x1b?, 0xc002c0df74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c0dde8 sp=0xc002c0ddc8 pc=0x4777ae +runtime.selectgo(0xc002c0dfb0, 0xc002c0df70, 0x0?, 0x0, 0xc002000000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c0df38 sp=0xc002c0dde8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002c0dfe0 sp=0xc002c0df38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c0dfe8 sp=0xc002c0dfe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1367 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1464 [select]: +runtime.gopark(0xc002c0efb0?, 0x2?, 0xb0?, 0xed?, 0xc002c0eee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c0ed58 sp=0xc002c0ed38 pc=0x4777ae +runtime.selectgo(0xc002c0efb0, 0xc002c0eee0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c0eea8 sp=0xc002c0ed58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002c0efe0 sp=0xc002c0eea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c0efe8 sp=0xc002c0efe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1465 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c21d60 sp=0xc002c21d40 pc=0x4777ae +runtime.chanrecv(0xc002bf68a0, 0xc002c21e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c21dd8 sp=0xc002c21d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c21e00 sp=0xc002c21dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002c21fe0 sp=0xc002c21e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c21fe8 sp=0xc002c21fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1473 [select]: +runtime.gopark(0xc002bae890?, 0x2?, 0x98?, 0xe7?, 0xc002bae83c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002bae6a8 sp=0xc002bae688 pc=0x4777ae +runtime.selectgo(0xc002bae890, 0xc002bae838, 0x7f429a48af00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002bae7f8 sp=0xc002bae6a8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002bfceb0, {0xc002bfedc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002bae8c0 sp=0xc002bae7f8 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002bfceb0, {0xc002bfedc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002bae958 sp=0xc002bae8c0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002bfecf0, {0xc002bfedc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002bae9a0 sp=0xc002bae958 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002bfecf0}, {0xc002bfedc0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002bae9e8 sp=0xc002bae9a0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8a360, {0xc002bfedc0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002baea58 sp=0xc002bae9e8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002bfedb0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002baeb28 sp=0xc002baea58 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc00236c8c0?, 0xc001d8a360, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002baec18 sp=0xc002baeb28 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002bfedb0, {0x7f424347ae80, 0x779c980}, 0x5?, {0x0, 0x0}, {0x43dcf60, 0xc002bfd270}, 0xc0025cdad0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002baecd0 sp=0xc002baec18 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001670870, {0x43dcf60?, 0xc002bfd270}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002baef08 sp=0xc002baecd0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002bfef00, {0x43dcf60, 0xc002bfd270}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002baeff0 sp=0xc002baef08 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002bf8580, {0x43dcf60, 0xc002bfd270}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002baf028 sp=0xc002baeff0 pc=0x1fd94b2 +google.golang.org/grpc/reflection/grpc_reflection_v1alpha.(*serverReflectionServerReflectionInfoServer).Recv(0xc0026890e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:139 +0x66 fp=0xc002baf068 sp=0xc002baf028 pc=0x130d8e6 +google.golang.org/grpc/reflection.v1AlphaServerStreamAdapter.Recv({{0x5b51038?, 0xc0026890e0?}}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:51 +0x3c fp=0xc002baf0a8 sp=0xc002baf068 pc=0x134d99c +google.golang.org/grpc/reflection.(*v1AlphaServerStreamAdapter).Recv(0xc0026890f0) + :1 +0x46 fp=0xc002baf0e0 sp=0xc002baf0a8 pc=0x13521e6 +google.golang.org/grpc/reflection.(*serverReflectionServer).ServerReflectionInfo(0xc0025cdc50, {0x5b51368, 0xc0026890f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/serverreflection.go:259 +0xc5 fp=0xc002baf320 sp=0xc002baf0e0 pc=0x13509a5 +google.golang.org/grpc/reflection.v1AlphaServerImpl.ServerReflectionInfo(...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/adapt.go:39 +google.golang.org/grpc/reflection.(*v1AlphaServerImpl).ServerReflectionInfo(0xc0026e2fa0, {0x5b51038, 0xc0026890e0}) + :1 +0xaf fp=0xc002baf370 sp=0xc002baf320 pc=0x1351fef +google.golang.org/grpc/reflection/grpc_reflection_v1alpha._ServerReflection_ServerReflectionInfo_Handler({0x41d18a0?, 0xc0026e2fa0}, {0x5b4a6f0?, 0xc002bf8580}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go:120 +0xb6 fp=0xc002baf3b0 sp=0xc002baf370 pc=0x130d776 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x41d18a0, 0xc0026e2fa0}, {0x5b4ab30, 0xc002bfef00}, 0xc000dc3128?, 0x47efbf0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002baf400 sp=0xc002baf3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x41d18a0, 0xc0026e2fa0}, {0x5b4ab30, 0xc002bfef00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002baf490 sp=0xc002baf400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x41d18a0, 0xc0026e2fa0}, {0x5b492f8, 0xc001670870}, 0xc000dc3128, 0xc00236c240) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002baf7d0 sp=0xc002baf490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x41d18a0, 0xc0026e2fa0}, {0x5b492f8, 0xc001670870}, 0x3f252a0?, 0xc002689010?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002baf848 sp=0xc002baf7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d8a360, 0xc0025cdce0, 0x770cb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002bafdb0 sp=0xc002baf848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001d8a360, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002baff28 sp=0xc002bafdb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002baffe0 sp=0xc002baff28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002baffe8 sp=0xc002baffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1398 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1469 [select]: +runtime.gopark(0xc002c0afb0?, 0x2?, 0xb4?, 0x1b?, 0xc002c0af74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c0ade8 sp=0xc002c0adc8 pc=0x4777ae +runtime.selectgo(0xc002c0afb0, 0xc002c0af70, 0xc002bf60c0?, 0x0, 0xc002780810?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c0af38 sp=0xc002c0ade8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002c0afe0 sp=0xc002c0af38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c0afe8 sp=0xc002c0afe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1367 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1470 [select]: +runtime.gopark(0xc002c1dfb0?, 0x2?, 0xb0?, 0xdd?, 0xc002c1dee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c1dd58 sp=0xc002c1dd38 pc=0x4777ae +runtime.selectgo(0xc002c1dfb0, 0xc002c1dee0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c1dea8 sp=0xc002c1dd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002c1dfe0 sp=0xc002c1dea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c1dfe8 sp=0xc002c1dfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1471 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c08d60 sp=0xc002c08d40 pc=0x4777ae +runtime.chanrecv(0xc002bf6cc0, 0xc002c08e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c08dd8 sp=0xc002c08d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c08e00 sp=0xc002c08dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002c08fe0 sp=0xc002c08e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c08fe8 sp=0xc002c08fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1367 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1478 [chan receive]: +runtime.gopark(0x7769700?, 0x460a1ee?, 0x5?, 0x0?, 0x462ff63?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0020f3b90 sp=0xc0020f3b70 pc=0x4777ae +runtime.chanrecv(0xc002666b40, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0020f3c08 sp=0xc0020f3b90 pc=0x441325 +runtime.chanrecv1(0x4b05a9?, 0xc0020f3c58?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0020f3c30 sp=0xc0020f3c08 pc=0x440f72 +github.com/rancher/opni/pkg/util/future.(*future[...]).Get(...) + /home/ec2-user/opni/pkg/util/future/future.go:61 +github.com/rancher/opni/pkg/util/future.(*future[...]).Get() + /home/ec2-user/opni/pkg/util/future/future.go:60 +0x5a fp=0xc0020f3c68 sp=0xc0020f3c30 pc=0x201331a +github.com/rancher/opni/pkg/agent/v2.(*Agent).runGatewayClient(0xc0025ee340, {0x5b426e0, 0xc002c2e550}) + /home/ec2-user/opni/pkg/agent/v2/agent.go:522 +0x34c fp=0xc0020f3f50 sp=0xc0020f3c68 pc=0x21c052c +github.com/rancher/opni/pkg/agent/v2.(*Agent).ListenAndServe.func3() + /home/ec2-user/opni/pkg/agent/v2/agent.go:446 +0x47 fp=0xc0020f3fa0 sp=0xc0020f3f50 pc=0x21bf167 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc0020f3fe0 sp=0xc0020f3fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0020f3fe8 sp=0xc0020f3fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1320 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1479 [select]: +runtime.gopark(0xc002c09fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002c09f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c09de8 sp=0xc002c09dc8 pc=0x4777ae +runtime.selectgo(0xc002c09fb0, 0xc002c09f70, 0x21bf167?, 0x0, 0x5b426e0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c09f38 sp=0xc002c09de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002c09fe0 sp=0xc002c09f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c09fe8 sp=0xc002c09fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1478 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1422 [select]: +runtime.gopark(0xc0029b4fb0?, 0x2?, 0xb0?, 0x4d?, 0xc0029b4ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b4d58 sp=0xc0029b4d38 pc=0x4777ae +runtime.selectgo(0xc0029b4fb0, 0xc0029b4ee0, 0xc000a8ef00?, 0x0, 0x1174725?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029b4ea8 sp=0xc0029b4d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc0029b4fe0 sp=0xc0029b4ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b4fe8 sp=0xc0029b4fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1478 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1423 [chan receive]: +runtime.gopark(0xc00160e7c0?, 0xc0016190e0?, 0x70?, 0x7?, 0xc001340780?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b2d60 sp=0xc0029b2d40 pc=0x4777ae +runtime.chanrecv(0xc0022fbec0, 0xc0029b2e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0029b2dd8 sp=0xc0029b2d60 pc=0x441325 +runtime.chanrecv1(0xc000770a10?, 0xc00086d110?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0029b2e00 sp=0xc0029b2dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0029b2fe0 sp=0xc0029b2e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b2fe8 sp=0xc0029b2fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1478 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1424 [select]: +runtime.gopark(0xc002c1afb0?, 0x2?, 0xa9?, 0x5?, 0xc002c1af2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c1ada8 sp=0xc002c1ad88 pc=0x4777ae +runtime.selectgo(0xc002c1afb0, 0xc002c1af28, 0x4b0571?, 0x0, 0x11847d3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c1aef8 sp=0xc002c1ada8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002c1afe0 sp=0xc002c1aef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c1afe8 sp=0xc002c1afe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1425 [select]: +runtime.gopark(0xc002c8bf90?, 0x2?, 0xff?, 0xff?, 0xc002c8beac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c8bd28 sp=0xc002c8bd08 pc=0x4777ae +runtime.selectgo(0xc002c8bf90, 0xc002c8bea8, 0x4b0571?, 0x0, 0xc000f43718?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c8be78 sp=0xc002c8bd28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc002949680) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002c8bfc0 sp=0xc002c8be78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002c8bfe0 sp=0xc002c8bfc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c8bfe8 sp=0xc002c8bfe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1490 [select]: +runtime.gopark(0xc002ec7f40?, 0x2?, 0xb4?, 0x1b?, 0xc002ec7ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ec7d48 sp=0xc002ec7d28 pc=0x4777ae +runtime.selectgo(0xc002ec7f40, 0xc002ec7ec8, 0xc002711470?, 0x0, 0x5655434a55573374?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ec7e98 sp=0xc002ec7d48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002846d20}, 0xc002949710, 0x0, 0xc00245ee40, 0xc002577b50) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002ec7f70 sp=0xc002ec7e98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002ec7fe0 sp=0xc002ec7f70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ec7fe8 sp=0xc002ec7fe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1491 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0xbe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c1be20 sp=0xc002c1be00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c1be98 sp=0xc002c1be20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x2166b62?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c1bec0 sp=0xc002c1be98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c1bfe0 sp=0xc002c1bec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c1bfe8 sp=0xc002c1bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1492 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x7e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c87e20 sp=0xc002c87e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c87e98 sp=0xc002c87e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x5b058a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c87ec0 sp=0xc002c87e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c87fe0 sp=0xc002c87ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c87fe8 sp=0xc002c87fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1493 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x8e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c88e20 sp=0xc002c88e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c88e98 sp=0xc002c88e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c88ec0 sp=0xc002c88e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c88fe0 sp=0xc002c88ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c88fe8 sp=0xc002c88fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1494 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x9e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c89e20 sp=0xc002c89e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c89e98 sp=0xc002c89e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x4166b20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c89ec0 sp=0xc002c89e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c89fe0 sp=0xc002c89ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c89fe8 sp=0xc002c89fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1495 [select]: +runtime.gopark(0xc002c8afb0?, 0x2?, 0xb4?, 0x1b?, 0xc002c8af74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c8ade8 sp=0xc002c8adc8 pc=0x4777ae +runtime.selectgo(0xc002c8afb0, 0xc002c8af70, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c8af38 sp=0xc002c8ade8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002c8afe0 sp=0xc002c8af38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c8afe8 sp=0xc002c8afe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1478 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1496 [select]: +runtime.gopark(0xc002d0bfb0?, 0x2?, 0x0?, 0x0?, 0xc002d0bee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d0bd58 sp=0xc002d0bd38 pc=0x4777ae +runtime.selectgo(0xc002d0bfb0, 0xc002d0bee0, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d0bea8 sp=0xc002d0bd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002d0bfe0 sp=0xc002d0bea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d0bfe8 sp=0xc002d0bfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1478 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1497 [chan receive]: +runtime.gopark(0xc0009f2430?, 0xc000992790?, 0xd0?, 0x27?, 0xc000992810?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029b6d60 sp=0xc0029b6d40 pc=0x4777ae +runtime.chanrecv(0xc002858300, 0xc0029b6e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0029b6dd8 sp=0xc0029b6d60 pc=0x441325 +runtime.chanrecv1(0xc000807a00?, 0xc0009d3420?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0029b6e00 sp=0xc0029b6dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc0029b6fe0 sp=0xc0029b6e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029b6fe8 sp=0xc0029b6fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1478 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1498 [select]: +runtime.gopark(0xc002d07fb0?, 0x2?, 0x0?, 0x7e?, 0xc002d07f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d07da8 sp=0xc002d07d88 pc=0x4777ae +runtime.selectgo(0xc002d07fb0, 0xc002d07f28, 0x4b0571?, 0x0, 0x11847d3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d07ef8 sp=0xc002d07da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002d07fe0 sp=0xc002d07ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d07fe8 sp=0xc002d07fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1499 [select]: +runtime.gopark(0xc002d09f90?, 0x2?, 0xb4?, 0x1b?, 0xc002d09eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d09d28 sp=0xc002d09d08 pc=0x4777ae +runtime.selectgo(0xc002d09f90, 0xc002d09ea8, 0x4b0571?, 0x0, 0x4b0571?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d09e78 sp=0xc002d09d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc002949800) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002d09fc0 sp=0xc002d09e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002d09fe0 sp=0xc002d09fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d09fe8 sp=0xc002d09fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1500 [select]: +runtime.gopark(0xc002babf40?, 0x2?, 0xb4?, 0x1b?, 0xc002babecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002babd48 sp=0xc002babd28 pc=0x4777ae +runtime.selectgo(0xc002babf40, 0xc002babec8, 0x4b0571?, 0x0, 0x8ab66c?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002babe98 sp=0xc002babd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002847680}, 0xc002949890, 0x0, 0xc0022ee1e0, 0xc0029088c0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002babf70 sp=0xc002babe98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002babfe0 sp=0xc002babf70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002babfe8 sp=0xc002babfe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1501 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc0028581e0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d05e20 sp=0xc002d05e00 pc=0x4777ae +runtime.chanrecv(0xc0028581e0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d05e98 sp=0xc002d05e20 pc=0x441325 +runtime.chanrecv1(0xc002849cb0?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d05ec0 sp=0xc002d05e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d05fe0 sp=0xc002d05ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d05fe8 sp=0xc002d05fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1502 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc0028581e0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d0ae20 sp=0xc002d0ae00 pc=0x4777ae +runtime.chanrecv(0xc0028581e0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d0ae98 sp=0xc002d0ae20 pc=0x441325 +runtime.chanrecv1(0xc002849cb0?, 0x6a55575255575268?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d0aec0 sp=0xc002d0ae98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d0afe0 sp=0xc002d0aec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d0afe8 sp=0xc002d0afe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1503 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x3e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cb3e20 sp=0xc002cb3e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cb3e98 sp=0xc002cb3e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x2166b62?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cb3ec0 sp=0xc002cb3e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cb3fe0 sp=0xc002cb3ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cb3fe8 sp=0xc002cb3fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1504 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d04e20 sp=0xc002d04e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d04e98 sp=0xc002d04e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x2005f24?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d04ec0 sp=0xc002d04e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d04fe0 sp=0xc002d04ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d04fe8 sp=0xc002d04fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1505 [select]: +runtime.gopark(0xc002b95c98?, 0x2?, 0x67?, 0x6a?, 0xc002b95bb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002b95a28 sp=0xc002b95a08 pc=0x4777ae +runtime.selectgo(0xc002b95c98, 0xc002b95bb0, 0xc002574ea0?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002b95b78 sp=0xc002b95a28 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc0024e9220) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002b95cc8 sp=0xc002b95b78 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc001c96b40, {0x5b426a8, 0xc002849cb0}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002b95ee8 sp=0xc002b95cc8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Splice.func2() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:190 +0x6b fp=0xc002b95fe0 sp=0xc002b95ee8 pc=0x136faab +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002b95fe8 sp=0xc002b95fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Splice in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:189 +0x61a + +goroutine 1486 [semacquire]: +runtime.gopark(0x473391?, 0xc002771238?, 0xa0?, 0x57?, 0x448485?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0027711c8 sp=0xc0027711a8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +runtime.semacquire1(0xc002f90290, 0x58?, 0x1, 0x0, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:160 +0x218 fp=0xc002771230 sp=0xc0027711c8 pc=0x48a478 +sync.runtime_Semacquire(0xc002f90290?) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:62 +0x25 fp=0xc002771268 sp=0xc002771230 pc=0x4a8fc5 +sync.(*WaitGroup).Wait(0xc002f90288) + /home/ec2-user/sdk/go1.21.0/src/sync/waitgroup.go:116 +0xa5 fp=0xc002771298 sp=0xc002771268 pc=0x4c06e5 +golang.org/x/sync/errgroup.(*Group).Wait(0xc002f90280) + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:53 +0x38 fp=0xc0027712c8 sp=0xc002771298 pc=0x20071d8 +github.com/rancher/opni/pkg/management.NewServer.unknownServiceHandler.func3({0xc000ef57c0?, 0x5b426a8?}, {0x5b4ab30?, 0xc002c31a40}) + /home/ec2-user/opni/pkg/management/extensions.go:567 +0xa85 fp=0xc002771508 sp=0xc0027712c8 pc=0x231d105 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x0, 0x0}, {0x5b492f8, 0xc001670e10}, 0xc001e465d0, 0xc00158a890) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002771848 sp=0xc002771508 pc=0x12b0e02 +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c1e0, {0x5b54e00, 0xc002c26d00}, 0xc001d8afc0, 0x0, 0xc001ded2a0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002771db0 sp=0xc002771848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c1e0, {0x5b54e00, 0xc002c26d00}, 0xc001d8afc0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1747 +0xf15 fp=0xc002771f28 sp=0xc002771db0 pc=0x11846f5 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002771fe0 sp=0xc002771f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002771fe8 sp=0xc002771fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1484 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1457 [select]: +runtime.gopark(0xc002bb3f40?, 0x2?, 0xb4?, 0x1b?, 0xc002bb3ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002bb3d48 sp=0xc002bb3d28 pc=0x4777ae +runtime.selectgo(0xc002bb3f40, 0xc002bb3ec8, 0xc002849bc0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002bb3e98 sp=0xc002bb3d48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002669090}, 0xc0015ec2d0, 0x0, 0xc00251db00, 0xc002d00550) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002bb3f70 sp=0xc002bb3e98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002bb3fe0 sp=0xc002bb3f70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002bb3fe8 sp=0xc002bb3fe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1454 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1506 [chan receive]: +runtime.gopark(0xc00266d770?, 0x4b05a9?, 0x50?, 0xbe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00234be20 sp=0xc00234be00 pc=0x4777ae +runtime.chanrecv(0xc002666480, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc00234be98 sp=0xc00234be20 pc=0x441325 +runtime.chanrecv1(0xc00266da40?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc00234bec0 sp=0xc00234be98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc00234bfe0 sp=0xc00234bec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00234bfe8 sp=0xc00234bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1454 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1507 [chan receive]: +runtime.gopark(0xc00266d770?, 0x4b05a9?, 0x50?, 0x6e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017d6e20 sp=0xc0017d6e00 pc=0x4777ae +runtime.chanrecv(0xc002666480, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0017d6e98 sp=0xc0017d6e20 pc=0x441325 +runtime.chanrecv1(0xc00266da40?, 0x4166b20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0017d6ec0 sp=0xc0017d6e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0017d6fe0 sp=0xc0017d6ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017d6fe8 sp=0xc0017d6fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1454 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1508 [chan receive]: +runtime.gopark(0xc00266d770?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c85e20 sp=0xc002c85e00 pc=0x4777ae +runtime.chanrecv(0xc002666480, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c85e98 sp=0xc002c85e20 pc=0x441325 +runtime.chanrecv1(0xc00266da40?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c85ec0 sp=0xc002c85e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c85fe0 sp=0xc002c85ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c85fe8 sp=0xc002c85fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1454 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1509 [chan receive]: +runtime.gopark(0xc00266d770?, 0x4b05a9?, 0x50?, 0x6e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c86e20 sp=0xc002c86e00 pc=0x4777ae +runtime.chanrecv(0xc002666480, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c86e98 sp=0xc002c86e20 pc=0x441325 +runtime.chanrecv1(0xc00266da40?, 0x5b058a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c86ec0 sp=0xc002c86e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c86fe0 sp=0xc002c86ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c86fe8 sp=0xc002c86fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1454 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1523 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc0028581e0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d08e20 sp=0xc002d08e00 pc=0x4777ae +runtime.chanrecv(0xc0028581e0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d08e98 sp=0xc002d08e20 pc=0x441325 +runtime.chanrecv1(0xc002849cb0?, 0x1000000010000?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d08ec0 sp=0xc002d08e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d08fe0 sp=0xc002d08ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d08fe8 sp=0xc002d08fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1482 [select]: +runtime.gopark(0xc002c5de48?, 0x2?, 0x0?, 0x0?, 0xc002c5ddcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c5dc30 sp=0xc002c5dc10 pc=0x4777ae +runtime.selectgo(0xc002c5de48, 0xc002c5ddc8, 0xc002c5ddd8?, 0x0, 0xc001d6f418?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c5dd80 sp=0xc002c5dc30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002c2ec30, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc002c5de78 sp=0xc002c5dd80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0026af880) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc002c5df30 sp=0xc002c5de78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc002c5dfe0 sp=0xc002c5df30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c5dfe8 sp=0xc002c5dfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1481 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1483 [select]: +runtime.gopark(0xc002c1ef70?, 0x4?, 0xf8?, 0xeb?, 0xc002c1edd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c1ec00 sp=0xc002c1ebe0 pc=0x4777ae +runtime.selectgo(0xc002c1ef70, 0xc002c1edd0, 0x40a16a0?, 0x0, 0xc0026f2dd0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c1ed50 sp=0xc002c1ec00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002c26d00) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc002c1efc0 sp=0xc002c1ed50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc002c1efe0 sp=0xc002c1efc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c1efe8 sp=0xc002c1efe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1481 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1484 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c3d7d8 sp=0xc002c3d7b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002c3d810 sp=0xc002c3d7d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c44c8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002c3d830 sp=0xc002c3d810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc002c19620, 0xc002c3e000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002c3d878 sp=0xc002c3d830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc002c19600, {0xc002c3e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc002c3d968 sp=0xc002c3d878 pc=0x50e5a5 +net.(*netFD).Read(0xc002c19600, {0xc002c3e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002c3d9c8 sp=0xc002c3d968 pc=0x89342b +net.(*conn).Read(0xc001d6f418, {0xc002c3e000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc002c3da50 sp=0xc002c3d9c8 pc=0x8b176d +net.(*TCPConn).Read(0xc001d6f418, {0xc002c3e000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc002c3da98 sp=0xc002c3da50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc002574fc0, {0xc002686820, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002c3db58 sp=0xc002c3da98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002574fc0}, {0xc002686820, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002c3dba0 sp=0xc002c3db58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc002686820, 0x9, 0x9}, {0x5b0f0a0, 0xc002574fc0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002c3dc40 sp=0xc002c3dba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0026867e0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002c3dd78 sp=0xc002c3dc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002c26d00, 0xc000b3d0e0?, 0x28?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc002c3dec8 sp=0xc002c3dd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c1e0, {0x5b54e00?, 0xc002c26d00}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc002c3df80 sp=0xc002c3dec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc002c3dfe0 sp=0xc002c3df80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c3dfe8 sp=0xc002c3dfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1481 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +goroutine 1522 [select]: +runtime.gopark(0xc002c153f8?, 0x2?, 0xc0?, 0x72?, 0xc002c1537c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c151e8 sp=0xc002c151c8 pc=0x4777ae +runtime.selectgo(0xc002c153f8, 0xc002c15378, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c15338 sp=0xc002c151e8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002847590, {0xc002849db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002c15428 sp=0xc002c15338 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002847590, {0xc002849db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002c154c0 sp=0xc002c15428 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002849d70, {0xc002849db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002c15508 sp=0xc002c154c0 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002849d70}, {0xc002849db0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002c15550 sp=0xc002c15508 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c96a20, {0xc002849db0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002c155c0 sp=0xc002c15550 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002849da0, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002c15690 sp=0xc002c155c0 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001c96a20, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002c15780 sp=0xc002c15690 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002849da0, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc002fe0310}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002c15838 sp=0xc002c15780 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002713d40, {0x43f1c20?, 0xc002fe0310}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002c15a98 sp=0xc002c15838 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001c96534?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002c15af8 sp=0xc002c15a98 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001c96480, 0xc002c15cb0, 0xc002c15c80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002c15ba0 sp=0xc002c15af8 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001c96480, {0x43f1c20, 0xc002fe0310}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002c15ce0 sp=0xc002c15ba0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0021e7f80, {0x43f1c20, 0xc002fe0310}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc002c15ea8 sp=0xc002c15ce0 pc=0x12ac5b9 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectClient).Recv(0xc002908810) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:71 +0x66 fp=0xc002c15ee8 sp=0xc002c15ea8 pc=0x2003b86 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002c15fe0 sp=0xc002c15ee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c15fe8 sp=0xc002c15fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1505 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1511 [select]: +runtime.gopark(0xc0029bfaa8?, 0x2?, 0x67?, 0x6a?, 0xc0029bf9c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029bf838 sp=0xc0029bf818 pc=0x4777ae +runtime.selectgo(0xc0029bfaa8, 0xc0029bf9c0, 0xc0029bf9d0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0029bf988 sp=0xc0029bf838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc002650f00) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc0029bfad8 sp=0xc0029bf988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc001c58480, {0x5b426a8, 0xc00266da40}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc0029bfcf8 sp=0xc0029bfad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc0029bffe0 sp=0xc0029bfcf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029bffe8 sp=0xc0029bffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 1510 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 1436 [select]: +runtime.gopark(0xc002cacf90?, 0x2?, 0xb4?, 0x1b?, 0xc002caceac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cacd28 sp=0xc002cacd08 pc=0x4777ae +runtime.selectgo(0xc002cacf90, 0xc002cacea8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cace78 sp=0xc002cacd28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc0004f1b00) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002cacfc0 sp=0xc002cace78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002cacfe0 sp=0xc002cacfc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cacfe8 sp=0xc002cacfe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1437 [select]: +runtime.gopark(0xc002ecbf40?, 0x2?, 0xb4?, 0x1b?, 0xc002ecbecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ecbd48 sp=0xc002ecbd28 pc=0x4777ae +runtime.selectgo(0xc002ecbf40, 0xc002ecbec8, 0x4ad421?, 0x0, 0x1381849?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ecbe98 sp=0xc002ecbd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc0027ca5f0}, 0xc0004f1b90, 0x0, 0xc002047680, 0xc002857840) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002ecbf70 sp=0xc002ecbe98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002ecbfe0 sp=0xc002ecbf70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ecbfe8 sp=0xc002ecbfe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1438 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x3e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d83e20 sp=0xc002d83e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d83e98 sp=0xc002d83e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x4166b20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d83ec0 sp=0xc002d83e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d83fe0 sp=0xc002d83ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d83fe8 sp=0xc002d83fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1439 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d85e20 sp=0xc002d85e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d85e98 sp=0xc002d85e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x5b058a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d85ec0 sp=0xc002d85e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d85fe0 sp=0xc002d85ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d85fe8 sp=0xc002d85fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1440 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d84e20 sp=0xc002d84e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d84e98 sp=0xc002d84e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x2154feb?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d84ec0 sp=0xc002d84e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d84fe0 sp=0xc002d84ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d84fe8 sp=0xc002d84fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1441 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0xde?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cade20 sp=0xc002cade00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cade98 sp=0xc002cade20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cadec0 sp=0xc002cade98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cadfe0 sp=0xc002cadec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cadfe8 sp=0xc002cadfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1512 [select]: +runtime.gopark(0xc0017c7710?, 0x2?, 0x0?, 0x0?, 0xc0017c76bc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017c7528 sp=0xc0017c7508 pc=0x4777ae +runtime.selectgo(0xc0017c7710, 0xc0017c76b8, 0x7f429a499600?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0017c7678 sp=0xc0017c7528 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002668eb0, {0xc00266d8a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc0017c7740 sp=0xc0017c7678 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002668eb0, {0xc00266d8a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc0017c77d8 sp=0xc0017c7740 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00266d7a0, {0xc00266d8a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc0017c7820 sp=0xc0017c77d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc00266d7a0}, {0xc00266d8a0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc0017c7868 sp=0xc0017c7820 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c58360, {0xc00266d8a0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc0017c78d8 sp=0xc0017c7868 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc00266d890, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0017c79a8 sp=0xc0017c78d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001c58360, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc0017c7a98 sp=0xc0017c79a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc00266d890, {0x7f424347ae80, 0x779c980}, 0x463c97f?, {0x0, 0x0}, {0x43f1c20, 0xc0030d2070}, 0xc00267f098?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc0017c7b50 sp=0xc0017c7a98 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000b9c2d0, {0x43f1c20?, 0xc0030d2070}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc0017c7d88 sp=0xc0017c7b50 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc00266d9e0, {0x43f1c20, 0xc0030d2070}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc0017c7e70 sp=0xc0017c7d88 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002488b60, {0x43f1c20, 0xc0030d2070}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc0017c7ea8 sp=0xc0017c7e70 pc=0x1fd94b2 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectServer).Recv(0xc002d00490) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:140 +0x66 fp=0xc0017c7ee8 sp=0xc0017c7ea8 pc=0x2004046 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc0017c7fe0 sp=0xc0017c7ee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017c7fe8 sp=0xc0017c7fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1511 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1538 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc0027996c0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cafdd8 sp=0xc002cafdb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002722050, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc002cafe28 sp=0xc002cafdd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc002722040) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc002cafe58 sp=0xc002cafe28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc0020ace00, 0xc0027996c0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc002caffb0 sp=0xc002cafe58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc002caffe0 sp=0xc002caffb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002caffe8 sp=0xc002caffe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 1524 [chan receive]: +runtime.gopark(0x431af0?, 0x0?, 0x16?, 0x0?, 0xc0028581e0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c1fe20 sp=0xc002c1fe00 pc=0x4777ae +runtime.chanrecv(0xc0028581e0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c1fe98 sp=0xc002c1fe20 pc=0x441325 +runtime.chanrecv1(0xc002849cb0?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c1fec0 sp=0xc002c1fe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c1ffe0 sp=0xc002c1fec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c1ffe8 sp=0xc002c1ffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1525 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc0028581e0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cb1e20 sp=0xc002cb1e00 pc=0x4777ae +runtime.chanrecv(0xc0028581e0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cb1e98 sp=0xc002cb1e20 pc=0x441325 +runtime.chanrecv1(0xc002849cb0?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cb1ec0 sp=0xc002cb1e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cb1fe0 sp=0xc002cb1ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cb1fe8 sp=0xc002cb1fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1526 [select]: +runtime.gopark(0xc002cb0fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002cb0f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cb0de8 sp=0xc002cb0dc8 pc=0x4777ae +runtime.selectgo(0xc002cb0fb0, 0xc002cb0f70, 0xc000f427d0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cb0f38 sp=0xc002cb0de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002cb0fe0 sp=0xc002cb0f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cb0fe8 sp=0xc002cb0fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1478 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1527 [select]: +runtime.gopark(0xc002cb2fb0?, 0x2?, 0x0?, 0x0?, 0xc002cb2ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cb2d58 sp=0xc002cb2d38 pc=0x4777ae +runtime.selectgo(0xc002cb2fb0, 0xc002cb2ee0, 0x1381849?, 0x0, 0x2166b62?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cb2ea8 sp=0xc002cb2d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002cb2fe0 sp=0xc002cb2ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cb2fe8 sp=0xc002cb2fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1478 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1528 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d3bd60 sp=0xc002d3bd40 pc=0x4777ae +runtime.chanrecv(0xc002858780, 0xc002d3be60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d3bdd8 sp=0xc002d3bd60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d3be00 sp=0xc002d3bdd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002d3bfe0 sp=0xc002d3be00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d3bfe8 sp=0xc002d3bfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1478 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1539 [chan receive]: +runtime.gopark(0x3fa3c40?, 0xc00235d7a0?, 0x0?, 0x0?, 0xc002cc7f00?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cc7f08 sp=0xc002cc7ee8 pc=0x4777ae +runtime.chanrecv(0xc00235d7a0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cc7f80 sp=0xc002cc7f08 pc=0x441325 +runtime.chanrecv1(0xc0027caa00?, 0xc0018534a0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cc7fa8 sp=0xc002cc7f80 pc=0x440f72 +github.com/nats-io/nats%2ego.(*js).subscribe.func3() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:1817 +0x4a fp=0xc002cc7fe0 sp=0xc002cc7fa8 pc=0x1f4624a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cc7fe8 sp=0xc002cc7fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*js).subscribe in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:1816 +0x2c85 + +goroutine 1529 [select]: +runtime.gopark(0xc002d3cfb0?, 0x2?, 0x0?, 0xce?, 0xc002d3cf2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d3cda8 sp=0xc002d3cd88 pc=0x4777ae +runtime.selectgo(0xc002d3cfb0, 0xc002d3cf28, 0x0?, 0x0, 0x12a6d85?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d3cef8 sp=0xc002d3cda8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002d3cfe0 sp=0xc002d3cef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d3cfe8 sp=0xc002d3cfe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1530 [select]: +runtime.gopark(0xc002d3af90?, 0x2?, 0xb4?, 0x1b?, 0xc002d3aeac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d3ad28 sp=0xc002d3ad08 pc=0x4777ae +runtime.selectgo(0xc002d3af90, 0xc002d3aea8, 0x5d53d0?, 0x0, 0xc002847680?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d3ae78 sp=0xc002d3ad28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc002949a40) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002d3afc0 sp=0xc002d3ae78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002d3afe0 sp=0xc002d3afc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d3afe8 sp=0xc002d3afe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1531 [select]: +runtime.gopark(0xc002e13f40?, 0x2?, 0xb4?, 0x1b?, 0xc002e13ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e13d48 sp=0xc002e13d28 pc=0x4777ae +runtime.selectgo(0xc002e13f40, 0xc002e13ec8, 0xc001c96480?, 0x0, 0x136de51?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e13e98 sp=0xc002e13d48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002cba0f0}, 0xc002949ad0, 0x0, 0xc0022ef620, 0xc002caaf00) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002e13f70 sp=0xc002e13e98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002e13fe0 sp=0xc002e13f70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e13fe8 sp=0xc002e13fe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1532 [chan receive]: +runtime.gopark(0x70?, 0x0?, 0x70?, 0x0?, 0xc002858660?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d06e20 sp=0xc002d06e00 pc=0x4777ae +runtime.chanrecv(0xc002858660, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d06e98 sp=0xc002d06e20 pc=0x441325 +runtime.chanrecv1(0xc002ca3620?, 0x2106cfa?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d06ec0 sp=0xc002d06e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d06fe0 sp=0xc002d06ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d06fe8 sp=0xc002d06fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1533 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc002858660?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ccbe20 sp=0xc002ccbe00 pc=0x4777ae +runtime.chanrecv(0xc002858660, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002ccbe98 sp=0xc002ccbe20 pc=0x441325 +runtime.chanrecv1(0xc002ca3620?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002ccbec0 sp=0xc002ccbe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002ccbfe0 sp=0xc002ccbec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ccbfe8 sp=0xc002ccbfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1513 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d46cd0 sp=0xc002d46cb0 pc=0x4777ae +runtime.chanrecv(0xc002cce600, 0xc002d46f90, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d46d48 sp=0xc002d46cd0 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d46d70 sp=0xc002d46d48 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/stream.(*agentStreamExtensionServerImpl).Connect(0xc001e55380, {0x5b508a0, 0xc002d010a0}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/stream/plugin_agent.go:166 +0x1065 fp=0xc002d47370 sp=0xc002d46d70 pc=0x261cb65 +github.com/rancher/opni/pkg/apis/stream/v1._Stream_Connect_Handler({0x4229120?, 0xc001e55380}, {0x5b4a6f0?, 0xc002488e40}) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:121 +0xb6 fp=0xc002d473b0 sp=0xc002d47370 pc=0x2003ed6 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x4229120, 0xc001e55380}, {0x5b4ab30, 0xc002d1c9f0}, 0xc0012e53c8?, 0x47ee1d0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002d47400 sp=0xc002d473b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x4229120, 0xc001e55380}, {0x5b4ab30, 0xc002d1c9f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002d47490 sp=0xc002d47400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4229120, 0xc001e55380}, {0x5b492f8, 0xc000b9c3c0}, 0xc0012e53c8, 0xc00268c9c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002d477d0 sp=0xc002d47490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x4229120, 0xc001e55380}, {0x5b492f8, 0xc000b9c3c0}, 0x3f252a0?, 0xc002d00e00?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002d47848 sp=0xc002d477d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001c58c60, 0xc0027801b0, 0x770c800, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002d47db0 sp=0xc002d47848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc001853680, {0x5b54e00, 0xc002515860}, 0xc001c58c60, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002d47f28 sp=0xc002d47db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002d47fe0 sp=0xc002d47f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d47fe8 sp=0xc002d47fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1398 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1540 [select]: +runtime.gopark(0xc002daff70?, 0x2?, 0x1?, 0x0?, 0xc002dafe7c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002dafce0 sp=0xc002dafcc0 pc=0x4777ae +runtime.selectgo(0xc002daff70, 0xc002dafe78, 0xc0027ac930?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002dafe30 sp=0xc002dafce0 pc=0x48916b +github.com/rancher/opni/pkg/storage/jetstream.(*JetStreamStore).WatchCluster.func1() + /home/ec2-user/opni/pkg/storage/jetstream/cluster_store.go:137 +0x46c fp=0xc002daffe0 sp=0xc002dafe30 pc=0x249cecc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002daffe8 sp=0xc002daffe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage/jetstream.(*JetStreamStore).WatchCluster in goroutine 1174 + /home/ec2-user/opni/pkg/storage/jetstream/cluster_store.go:110 +0x31d + +goroutine 1534 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x8e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d38e20 sp=0xc002d38e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d38e98 sp=0xc002d38e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d38ec0 sp=0xc002d38e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d38fe0 sp=0xc002d38ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d38fe8 sp=0xc002d38fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1541 [select]: +runtime.gopark(0xc002cc9f80?, 0x2?, 0x0?, 0x0?, 0xc002cc9ec4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cc9d40 sp=0xc002cc9d20 pc=0x4777ae +runtime.selectgo(0xc002cc9f80, 0xc002cc9ec0, 0x4b0571?, 0x0, 0x5d8cc5?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cc9e90 sp=0xc002cc9d40 pc=0x48916b +github.com/rancher/opni/pkg/storage.(*watchContext[...]).watch(0x0) + /home/ec2-user/opni/pkg/storage/context.go:44 +0x189 fp=0xc002cc9fb0 sp=0xc002cc9e90 pc=0x22c2fa9 +github.com/rancher/opni/pkg/storage.NewWatchContext[...].func1() + /home/ec2-user/opni/pkg/storage/context.go:37 +0x45 fp=0xc002cc9fe0 sp=0xc002cc9fb0 pc=0x22c18a5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cc9fe8 sp=0xc002cc9fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/storage.NewWatchContext[...] in goroutine 1174 + /home/ec2-user/opni/pkg/storage/context.go:37 +0x225 + +goroutine 1535 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x9e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d39e20 sp=0xc002d39e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d39e98 sp=0xc002d39e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x1174725?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d39ec0 sp=0xc002d39e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d39fe0 sp=0xc002d39ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d39fe8 sp=0xc002d39fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1536 [select]: +runtime.gopark(0xc002c39c98?, 0x2?, 0x67?, 0x6a?, 0xc002c39bb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c39a28 sp=0xc002c39a08 pc=0x4777ae +runtime.selectgo(0xc002c39c98, 0xc002c39bb0, 0xc000bc4c00?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c39b78 sp=0xc002c39a28 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc0024e9360) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002c39cc8 sp=0xc002c39b78 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc001c97c20, {0x5b426a8, 0xc002ca3620}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002c39ee8 sp=0xc002c39cc8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Splice.func2() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:190 +0x6b fp=0xc002c39fe0 sp=0xc002c39ee8 pc=0x136faab +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c39fe8 sp=0xc002c39fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Splice in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:189 +0x61a + +goroutine 1542 [select]: +runtime.gopark(0xc002cc8fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002cc8f6c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cc8dd8 sp=0xc002cc8db8 pc=0x4777ae +runtime.selectgo(0xc002cc8fb0, 0xc002cc8f68, 0xc00235cf00?, 0x0, 0xc002046a80?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cc8f28 sp=0xc002cc8dd8 pc=0x48916b +context.(*cancelCtx).propagateCancel.func2() + /home/ec2-user/sdk/go1.21.0/src/context/context.go:505 +0xe5 fp=0xc002cc8fe0 sp=0xc002cc8f28 pc=0x5d5bc5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cc8fe8 sp=0xc002cc8fe0 pc=0x4ad421 +created by context.(*cancelCtx).propagateCancel in goroutine 1174 + /home/ec2-user/sdk/go1.21.0/src/context/context.go:504 +0x530 + +goroutine 1543 [select]: +runtime.gopark(0xc002ccafb0?, 0x2?, 0xb4?, 0x1b?, 0xc002ccaf74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ccade8 sp=0xc002ccadc8 pc=0x4777ae +runtime.selectgo(0xc002ccafb0, 0xc002ccaf70, 0xc00235c9c0?, 0x0, 0xc00235ca20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ccaf38 sp=0xc002ccade8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002ccafe0 sp=0xc002ccaf38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ccafe8 sp=0xc002ccafe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1174 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1544 [select]: +runtime.gopark(0xc002db9fb0?, 0x2?, 0x0?, 0x0?, 0xc002db9ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db9d58 sp=0xc002db9d38 pc=0x4777ae +runtime.selectgo(0xc002db9fb0, 0xc002db9ee0, 0x448855?, 0x0, 0x4166b20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002db9ea8 sp=0xc002db9d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002db9fe0 sp=0xc002db9ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db9fe8 sp=0xc002db9fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1174 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1545 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d86d60 sp=0xc002d86d40 pc=0x4777ae +runtime.chanrecv(0xc00235d740, 0xc002d86e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d86dd8 sp=0xc002d86d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d86e00 sp=0xc002d86dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002d86fe0 sp=0xc002d86e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d86fe8 sp=0xc002d86fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1174 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1514 [select]: +runtime.gopark(0xc002d80fb0?, 0x2?, 0x0?, 0x0?, 0xc002d80f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d80da8 sp=0xc002d80d88 pc=0x4777ae +runtime.selectgo(0xc002d80fb0, 0xc002d80f28, 0x4b0571?, 0x0, 0x11847d3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d80ef8 sp=0xc002d80da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002d80fe0 sp=0xc002d80ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d80fe8 sp=0xc002d80fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1513 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1515 [select]: +runtime.gopark(0xc002d81f90?, 0x2?, 0xff?, 0xff?, 0xc002d81eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d81d28 sp=0xc002d81d08 pc=0x4777ae +runtime.selectgo(0xc002d81f90, 0xc002d81ea8, 0x5d5276?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d81e78 sp=0xc002d81d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc0015ec780) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002d81fc0 sp=0xc002d81e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002d81fe0 sp=0xc002d81fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d81fe8 sp=0xc002d81fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1513 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1516 [select]: +runtime.gopark(0xc002e0ff40?, 0x2?, 0xb4?, 0x1b?, 0xc002e0fecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e0fd48 sp=0xc002e0fd28 pc=0x4777ae +runtime.selectgo(0xc002e0ff40, 0xc002e0fec8, 0x4ad421?, 0x0, 0x1381849?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e0fe98 sp=0xc002e0fd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002669810}, 0xc0015ec810, 0x0, 0xc002d20de0, 0xc002d01160) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002e0ff70 sp=0xc002e0fe98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002e0ffe0 sp=0xc002e0ff70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e0ffe8 sp=0xc002e0ffe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1513 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1487 [select]: +runtime.gopark(0xc002d3efb0?, 0x2?, 0xb4?, 0x1b?, 0xc002d3ef74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d3ede8 sp=0xc002d3edc8 pc=0x4777ae +runtime.selectgo(0xc002d3efb0, 0xc002d3ef70, 0x0?, 0x0, 0x8d0f0a?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d3ef38 sp=0xc002d3ede8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002d3efe0 sp=0xc002d3ef38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d3efe8 sp=0xc002d3efe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1486 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1517 [chan receive]: +runtime.gopark(0xc002d1c7e0?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db5e20 sp=0xc002db5e00 pc=0x4777ae +runtime.chanrecv(0xc002666720, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002db5e98 sp=0xc002db5e20 pc=0x441325 +runtime.chanrecv1(0xc002d1ca50?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002db5ec0 sp=0xc002db5e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002db5fe0 sp=0xc002db5ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db5fe8 sp=0xc002db5fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1513 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1488 [select]: +runtime.gopark(0xc002c6dfb0?, 0x2?, 0x0?, 0x0?, 0xc002c6dee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c6dd58 sp=0xc002c6dd38 pc=0x4777ae +runtime.selectgo(0xc002c6dfb0, 0xc002c6dee0, 0x1381849?, 0x0, 0x6a55575255575268?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c6dea8 sp=0xc002c6dd58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002c6dfe0 sp=0xc002c6dea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c6dfe8 sp=0xc002c6dfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1486 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1489 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d3dd60 sp=0xc002d3dd40 pc=0x4777ae +runtime.chanrecv(0xc002bf7aa0, 0xc002d3de60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d3ddd8 sp=0xc002d3dd60 pc=0x441325 +runtime.chanrecv1(0x1d?, 0xa?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d3de00 sp=0xc002d3ddd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002d3dfe0 sp=0xc002d3de00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d3dfe8 sp=0xc002d3dfe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1486 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1518 [chan receive]: +runtime.gopark(0xc002d1c7e0?, 0x4b05a9?, 0x50?, 0xbe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c6be20 sp=0xc002c6be00 pc=0x4777ae +runtime.chanrecv(0xc002666720, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c6be98 sp=0xc002c6be20 pc=0x441325 +runtime.chanrecv1(0xc002d1ca50?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c6bec0 sp=0xc002c6be98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c6bfe0 sp=0xc002c6bec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c6bfe8 sp=0xc002c6bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1513 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1537 [select]: +runtime.gopark(0xc002c3b3f8?, 0x2?, 0xc0?, 0x72?, 0xc002c3b37c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c3b1e8 sp=0xc002c3b1c8 pc=0x4777ae +runtime.selectgo(0xc002c3b3f8, 0xc002c3b378, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c3b338 sp=0xc002c3b1e8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002cba000, {0xc002ca3720, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002c3b428 sp=0xc002c3b338 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002cba000, {0xc002ca3720, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002c3b4c0 sp=0xc002c3b428 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002ca36e0, {0xc002ca3720, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002c3b508 sp=0xc002c3b4c0 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002ca36e0}, {0xc002ca3720, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002c3b550 sp=0xc002c3b508 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c97b00, {0xc002ca3720, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002c3b5c0 sp=0xc002c3b550 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002ca3710, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002c3b690 sp=0xc002c3b5c0 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001c97b00, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002c3b780 sp=0xc002c3b690 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002ca3710, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc0027adb90}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002c3b838 sp=0xc002c3b780 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002ca65b0, {0x43f1c20?, 0xc0027adb90}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002c3ba98 sp=0xc002c3b838 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001c97854?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002c3baf8 sp=0xc002c3ba98 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001c977a0, 0xc002c3bcb0, 0xc002c3bc80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002c3bba0 sp=0xc002c3baf8 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001c977a0, {0x43f1c20, 0xc0027adb90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002c3bce0 sp=0xc002c3bba0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc002571500, {0x43f1c20, 0xc0027adb90}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc002c3bea8 sp=0xc002c3bce0 pc=0x12ac5b9 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectClient).Recv(0xc002caaea0) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:71 +0x66 fp=0xc002c3bee8 sp=0xc002c3bea8 pc=0x2003b86 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002c3bfe0 sp=0xc002c3bee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c3bfe8 sp=0xc002c3bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1536 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1519 [chan receive]: +runtime.gopark(0xc002d1c7e0?, 0x4b05a9?, 0x50?, 0xfe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d3fe20 sp=0xc002d3fe00 pc=0x4777ae +runtime.chanrecv(0xc002666720, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d3fe98 sp=0xc002d3fe20 pc=0x441325 +runtime.chanrecv1(0xc002d1ca50?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d3fec0 sp=0xc002d3fe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d3ffe0 sp=0xc002d3fec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d3ffe8 sp=0xc002d3ffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1513 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1576 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc002858660?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c6ae20 sp=0xc002c6ae00 pc=0x4777ae +runtime.chanrecv(0xc002858660, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c6ae98 sp=0xc002c6ae20 pc=0x441325 +runtime.chanrecv1(0xc002ca3620?, 0x100000001?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c6aec0 sp=0xc002c6ae98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c6afe0 sp=0xc002c6aec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c6afe8 sp=0xc002c6afe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1521 [select]: +runtime.gopark(0xc002c6cfb0?, 0x2?, 0x0?, 0xce?, 0xc002c6cf2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c6cda8 sp=0xc002c6cd88 pc=0x4777ae +runtime.selectgo(0xc002c6cfb0, 0xc002c6cf28, 0x1000000?, 0x0, 0x12a6d85?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c6cef8 sp=0xc002c6cda8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002c6cfe0 sp=0xc002c6cef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c6cfe8 sp=0xc002c6cfe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1554 [select]: +runtime.gopark(0xc002c69f90?, 0x2?, 0xff?, 0xff?, 0xc002c69eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c69d28 sp=0xc002c69d08 pc=0x4777ae +runtime.selectgo(0xc002c69f90, 0xc002c69ea8, 0x4b0571?, 0x0, 0x4b0571?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c69e78 sp=0xc002c69d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc0015ecfc0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002c69fc0 sp=0xc002c69e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002c69fe0 sp=0xc002c69fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c69fe8 sp=0xc002c69fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1555 [select]: +runtime.gopark(0xc002e3bf40?, 0x2?, 0xb4?, 0x1b?, 0xc002e3becc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e3bd48 sp=0xc002e3bd28 pc=0x4777ae +runtime.selectgo(0xc002e3bf40, 0xc002e3bec8, 0x4ad421?, 0x0, 0x1381849?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e3be98 sp=0xc002e3bd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002669ae0}, 0xc0015ed050, 0x0, 0xc002d212c0, 0xc002d01290) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002e3bf70 sp=0xc002e3be98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002e3bfe0 sp=0xc002e3bf70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e3bfe8 sp=0xc002e3bfe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1556 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc00235d560?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db7e20 sp=0xc002db7e00 pc=0x4777ae +runtime.chanrecv(0xc00235d560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002db7e98 sp=0xc002db7e20 pc=0x441325 +runtime.chanrecv1(0xc0027cd020?, 0x5b058a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002db7ec0 sp=0xc002db7e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002db7fe0 sp=0xc002db7ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db7fe8 sp=0xc002db7fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1557 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc00235d560?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db8e20 sp=0xc002db8e00 pc=0x4777ae +runtime.chanrecv(0xc00235d560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002db8e98 sp=0xc002db8e20 pc=0x441325 +runtime.chanrecv1(0xc0027cd020?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002db8ec0 sp=0xc002db8e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002db8fe0 sp=0xc002db8ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db8fe8 sp=0xc002db8fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1558 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x2e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d82e20 sp=0xc002d82e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002d82e98 sp=0xc002d82e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0xc002009ee0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002d82ec0 sp=0xc002d82e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002d82fe0 sp=0xc002d82ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d82fe8 sp=0xc002d82fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1559 [select]: +runtime.gopark(0xc002c37c98?, 0x2?, 0x67?, 0x6a?, 0xc002c37bb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c37a28 sp=0xc002c37a08 pc=0x4777ae +runtime.selectgo(0xc002c37c98, 0xc002c37bb0, 0xc002c37bc0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c37b78 sp=0xc002c37a28 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc002651220) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002c37cc8 sp=0xc002c37b78 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc001c59320, {0x5b426a8, 0xc0027cd020}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002c37ee8 sp=0xc002c37cc8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Splice.func2() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:190 +0x6b fp=0xc002c37fe0 sp=0xc002c37ee8 pc=0x136faab +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c37fe8 sp=0xc002c37fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Splice in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:189 +0x61a + +goroutine 1546 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0010ccd98 sp=0xc0010ccd78 pc=0x4777ae +runtime.chanrecv(0xc002e0af60, 0xc0010ccfb0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0010cce10 sp=0xc0010ccd98 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0010cce38 sp=0xc0010cce10 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/stream.(*gatewayStreamExtensionServerImpl).Connect(0xc001e7f570, {0x5b508a0, 0xc002c7f110}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/stream/plugin_gateway.go:157 +0xe51 fp=0xc0010cd370 sp=0xc0010cce38 pc=0x261f931 +github.com/rancher/opni/pkg/apis/stream/v1._Stream_Connect_Handler({0x4229080?, 0xc001e7f570}, {0x5b4a6f0?, 0xc002bf9680}) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:121 +0xb6 fp=0xc0010cd3b0 sp=0xc0010cd370 pc=0x2003ed6 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x4229080, 0xc001e7f570}, {0x5b4ab30, 0xc002c73c80}, 0xc001e472c0?, 0x47ee1d0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc0010cd400 sp=0xc0010cd3b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x4229080, 0xc001e7f570}, {0x5b4ab30, 0xc002c73c80}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc0010cd490 sp=0xc0010cd400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4229080, 0xc001e7f570}, {0x5b492f8, 0xc0016710e0}, 0xc001e472c0, 0xc0027a6c80) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc0010cd7d0 sp=0xc0010cd490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x4229080, 0xc001e7f570}, {0x5b492f8, 0xc0016710e0}, 0x3f252a0?, 0xc002c7eff0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc0010cd848 sp=0xc0010cd7d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0028a0c60, 0xc001267c80, 0x770c800, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc0010cddb0 sp=0xc0010cd848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0028a0c60, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc0010cdf28 sp=0xc0010cddb0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc0010cdfe0 sp=0xc0010cdf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0010cdfe8 sp=0xc0010cdfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1547 [select]: +runtime.gopark(0xc002dd32e0?, 0x4?, 0xb4?, 0x1b?, 0xc002dd3050?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002dd2e70 sp=0xc002dd2e50 pc=0x4777ae +runtime.selectgo(0xc002dd32e0, 0xc002dd3048, 0x24?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002dd2fc0 sp=0xc002dd2e70 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).SyncConfig(0xc00070a000, {0x5b50438?, 0xc002db02a0}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/admin.go:301 +0xb36 fp=0xc002dd3370 sp=0xc002dd2fc0 pc=0x395bbd6 +github.com/rancher/opni/plugins/alerting/apis/alertops._ConfigReconciler_SyncConfig_Handler({0x458d420?, 0xc00070a000}, {0x5b4a6f0?, 0xc002d9c200}) + /home/ec2-user/opni/plugins/alerting/apis/alertops/alertops_grpc.pb.go:468 +0xb6 fp=0xc002dd33b0 sp=0xc002dd3370 pc=0x2610b36 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x458d420, 0xc00070a000}, {0x5b4ab30, 0xc0027cd680}, 0xc00149fd10?, 0x47ec758) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002dd3400 sp=0xc002dd33b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x458d420, 0xc00070a000}, {0x5b4ab30, 0xc0027cd680}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002dd3490 sp=0xc002dd3400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x458d420, 0xc00070a000}, {0x5b492f8, 0xc00278a2d0}, 0xc00149fd10, 0xc0027225c0) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002dd37d0 sp=0xc002dd3490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x458d420, 0xc00070a000}, {0x5b492f8, 0xc00278a2d0}, 0x3f252a0?, 0xc002db01d0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002dd3848 sp=0xc002dd37d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0028a0fc0, 0xc001267890, 0x770c6c0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002dd3db0 sp=0xc002dd3848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc0028a0fc0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002dd3f28 sp=0xc002dd3db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002dd3fe0 sp=0xc002dd3f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002dd3fe8 sp=0xc002dd3fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1570 [semacquire]: +runtime.gopark(0x473391?, 0xc002c7b238?, 0x60?, 0x5b?, 0x448485?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c7b1c8 sp=0xc002c7b1a8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +runtime.semacquire1(0xc002d6e310, 0x58?, 0x1, 0x0, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:160 +0x218 fp=0xc002c7b230 sp=0xc002c7b1c8 pc=0x48a478 +sync.runtime_Semacquire(0xc002d6e310?) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:62 +0x25 fp=0xc002c7b268 sp=0xc002c7b230 pc=0x4a8fc5 +sync.(*WaitGroup).Wait(0xc002d6e308) + /home/ec2-user/sdk/go1.21.0/src/sync/waitgroup.go:116 +0xa5 fp=0xc002c7b298 sp=0xc002c7b268 pc=0x4c06e5 +golang.org/x/sync/errgroup.(*Group).Wait(0xc002d6e300) + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:53 +0x38 fp=0xc002c7b2c8 sp=0xc002c7b298 pc=0x20071d8 +github.com/rancher/opni/pkg/management.NewServer.unknownServiceHandler.func3({0xc000ef57c0?, 0x5b426a8?}, {0x5b4ab30?, 0xc002c721b0}) + /home/ec2-user/opni/pkg/management/extensions.go:567 +0xa85 fp=0xc002c7b508 sp=0xc002c7b2c8 pc=0x231d105 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x0, 0x0}, {0x5b492f8, 0xc001670f00}, 0xc001e467e0, 0xc00158a890) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002c7b848 sp=0xc002c7b508 pc=0x12b0e02 +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c1e0, {0x5b54e00, 0xc002c26d00}, 0xc001d8b440, 0x0, 0xc001ded2a0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002c7bdb0 sp=0xc002c7b848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c1e0, {0x5b54e00, 0xc002c26d00}, 0xc001d8b440, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1747 +0xf15 fp=0xc002c7bf28 sp=0xc002c7bdb0 pc=0x11846f5 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002c7bfe0 sp=0xc002c7bf28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c7bfe8 sp=0xc002c7bfe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1484 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1586 [select]: +runtime.gopark(0xc002d31aa8?, 0x2?, 0x67?, 0x6a?, 0xc002d319c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d31838 sp=0xc002d31818 pc=0x4777ae +runtime.selectgo(0xc002d31aa8, 0xc002d319c0, 0xc002d319d0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d31988 sp=0xc002d31838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc002651180) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002d31ad8 sp=0xc002d31988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc001c58d80, {0x5b426a8, 0xc002d1ca50}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002d31cf8 sp=0xc002d31ad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc002d31fe0 sp=0xc002d31cf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d31fe8 sp=0xc002d31fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 1520 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 1571 [select]: +runtime.gopark(0xc002cc4fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002cc4f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cc4de8 sp=0xc002cc4dc8 pc=0x4777ae +runtime.selectgo(0xc002cc4fb0, 0xc002cc4f70, 0x0?, 0x0, 0xc002000000?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cc4f38 sp=0xc002cc4de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002cc4fe0 sp=0xc002cc4f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cc4fe8 sp=0xc002cc4fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1570 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1572 [select]: +runtime.gopark(0xc002cc5fb0?, 0x2?, 0x0?, 0x0?, 0xc002cc5ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cc5d58 sp=0xc002cc5d38 pc=0x4777ae +runtime.selectgo(0xc002cc5fb0, 0xc002cc5ee0, 0x136de51?, 0x0, 0x2005f24?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cc5ea8 sp=0xc002cc5d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002cc5fe0 sp=0xc002cc5ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cc5fe8 sp=0xc002cc5fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1570 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1573 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cc6d60 sp=0xc002cc6d40 pc=0x4777ae +runtime.chanrecv(0xc002bf7e00, 0xc002cc6e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cc6dd8 sp=0xc002cc6d60 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cc6e00 sp=0xc002cc6dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002cc6fe0 sp=0xc002cc6e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cc6fe8 sp=0xc002cc6fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1570 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1574 [select]: +runtime.gopark(0xc002f392e0?, 0x4?, 0xb4?, 0x1b?, 0xc002f39050?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002f38e70 sp=0xc002f38e50 pc=0x4777ae +runtime.selectgo(0xc002f392e0, 0xc002f39048, 0x24?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002f38fc0 sp=0xc002f38e70 pc=0x48916b +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).SyncConfig(0xc00070a000, {0x5b50438?, 0xc002c7e0c0}) + /home/ec2-user/opni/plugins/alerting/pkg/alerting/admin.go:301 +0xb36 fp=0xc002f39370 sp=0xc002f38fc0 pc=0x395bbd6 +github.com/rancher/opni/plugins/alerting/apis/alertops._ConfigReconciler_SyncConfig_Handler({0x458d420?, 0xc00070a000}, {0x5b4a6f0?, 0xc002bf92c0}) + /home/ec2-user/opni/plugins/alerting/apis/alertops/alertops_grpc.pb.go:468 +0xb6 fp=0xc002f393b0 sp=0xc002f39370 pc=0x2610b36 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x458d420, 0xc00070a000}, {0x5b4ab30, 0xc002c72870}, 0xc001e46990?, 0x47ec758) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002f39400 sp=0xc002f393b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x458d420, 0xc00070a000}, {0x5b4ab30, 0xc002c72870}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002f39490 sp=0xc002f39400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x458d420, 0xc00070a000}, {0x5b492f8, 0xc001670ff0}, 0xc001e46990, 0xc002852540) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002f397d0 sp=0xc002f39490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x458d420, 0xc00070a000}, {0x5b492f8, 0xc001670ff0}, 0x3f252a0?, 0xc0026f3ab0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002f39848 sp=0xc002f397d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001d8bb00, 0xc001267890, 0x770c6c0, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002f39db0 sp=0xc002f39848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc0004ef0e0, {0x5b54e00, 0xc0017d04e0}, 0xc001d8bb00, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002f39f28 sp=0xc002f39db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002f39fe0 sp=0xc002f39f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002f39fe8 sp=0xc002f39fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 686 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1575 [select]: +runtime.gopark(0xc002c11710?, 0x2?, 0x0?, 0x0?, 0xc002c116bc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c11528 sp=0xc002c11508 pc=0x4777ae +runtime.selectgo(0xc002c11710, 0xc002c116b8, 0x7f429a49fc00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c11678 sp=0xc002c11528 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002669630, {0xc002d1c8b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002c11740 sp=0xc002c11678 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002669630, {0xc002d1c8b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002c117d8 sp=0xc002c11740 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002d1c810, {0xc002d1c8b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002c11820 sp=0xc002c117d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002d1c810}, {0xc002d1c8b0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002c11868 sp=0xc002c11820 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001c58c60, {0xc002d1c8b0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002c118d8 sp=0xc002c11868 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002d1c8a0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002c119a8 sp=0xc002c118d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001c58c60, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002c11a98 sp=0xc002c119a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002d1c8a0, {0x7f424347ae80, 0x779c980}, 0x40bed80?, {0x0, 0x0}, {0x43f1c20, 0xc00301cd20}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002c11b50 sp=0xc002c11a98 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc000b9c3c0, {0x43f1c20?, 0xc00301cd20}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002c11d88 sp=0xc002c11b50 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002d1c9f0, {0x43f1c20, 0xc00301cd20}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002c11e70 sp=0xc002c11d88 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002488e40, {0x43f1c20, 0xc00301cd20}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002c11ea8 sp=0xc002c11e70 pc=0x1fd94b2 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectServer).Recv(0xc002d010a0) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:140 +0x66 fp=0xc002c11ee8 sp=0xc002c11ea8 pc=0x2004046 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002c11fe0 sp=0xc002c11ee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c11fe8 sp=0xc002c11fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1586 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1577 [chan receive]: +runtime.gopark(0x70?, 0x0?, 0x70?, 0x0?, 0xc002858660?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c20e20 sp=0xc002c20e00 pc=0x4777ae +runtime.chanrecv(0xc002858660, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c20e98 sp=0xc002c20e20 pc=0x441325 +runtime.chanrecv1(0xc002ca3620?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c20ec0 sp=0xc002c20e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c20fe0 sp=0xc002c20ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c20fe8 sp=0xc002c20fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1578 [select]: +runtime.gopark(0xc002e21aa8?, 0x2?, 0x67?, 0x6a?, 0xc002e219c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e21838 sp=0xc002e21818 pc=0x4777ae +runtime.selectgo(0xc002e21aa8, 0xc002e219c0, 0xc002e219d0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e21988 sp=0xc002e21838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc0024e9180) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002e21ad8 sp=0xc002e21988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc001c6b9e0, {0x5b426a8, 0xc002849020}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002e21cf8 sp=0xc002e21ad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc002e21fe0 sp=0xc002e21cf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e21fe8 sp=0xc002e21fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 1579 [select]: +runtime.gopark(0xc002d2b3c0?, 0x2?, 0xc0?, 0x72?, 0xc002d2b344?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d2b1b0 sp=0xc002d2b190 pc=0x4777ae +runtime.selectgo(0xc002d2b3c0, 0xc002d2b340, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d2b300 sp=0xc002d2b1b0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002c2e730, {0xc002c30760, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002d2b3f0 sp=0xc002d2b300 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002c2e730, {0xc002c30760, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002d2b488 sp=0xc002d2b3f0 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c30720, {0xc002c30760, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002d2b4d0 sp=0xc002d2b488 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c30720}, {0xc002c30760, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002d2b518 sp=0xc002d2b4d0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8ac60, {0xc002c30760, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002d2b588 sp=0xc002d2b518 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c30750, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002d2b658 sp=0xc002d2b588 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001d8ac60, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002d2b748 sp=0xc002d2b658 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c30750, {0x7f424347ae80, 0x779c980}, 0xc002852540?, {0x0, 0x0}, {0x43f1c20, 0xc0027adf10}, 0x5b492f8?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002d2b800 sp=0xc002d2b748 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002c29380, {0x43f1c20?, 0xc0027adf10}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002d2ba60 sp=0xc002d2b800 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001d8aad4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002d2bac0 sp=0xc002d2ba60 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001d8aa20, 0xc002d2bc78, 0xc002d2bc48) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002d2bb68 sp=0xc002d2bac0 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8aa20, {0x43f1c20, 0xc0027adf10}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002d2bca8 sp=0xc002d2bb68 pc=0x119077e +github.com/rancher/opni/pkg/util/streams.(*ClientStreamWithContext).RecvMsg(0xc002bf8d40, {0x43f1c20, 0xc0027adf10}) + /home/ec2-user/opni/pkg/util/streams/streams.go:53 +0x52 fp=0xc002d2bce0 sp=0xc002d2bca8 pc=0x1fd8c72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0021e7380, {0x43f1c20, 0xc0027adf10}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc002d2bea8 sp=0xc002d2bce0 pc=0x12ac5b9 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectClient).Recv(0xc002577a70) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:71 +0x66 fp=0xc002d2bee8 sp=0xc002d2bea8 pc=0x2003b86 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002d2bfe0 sp=0xc002d2bee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d2bfe8 sp=0xc002d2bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1578 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1560 [select]: +runtime.gopark(0xc002d653f8?, 0x2?, 0xc0?, 0x72?, 0xc002d6537c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d651e8 sp=0xc002d651c8 pc=0x4777ae +runtime.selectgo(0xc002d653f8, 0xc002d65378, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d65338 sp=0xc002d651e8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0027cac80, {0xc0027cd120, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002d65428 sp=0xc002d65338 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0027cac80, {0xc0027cd120, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002d654c0 sp=0xc002d65428 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0027cd0e0, {0xc0027cd120, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002d65508 sp=0xc002d654c0 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0027cd0e0}, {0xc0027cd120, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002d65550 sp=0xc002d65508 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0028a0900, {0xc0027cd120, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002d655c0 sp=0xc002d65550 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0027cd110, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002d65690 sp=0xc002d655c0 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc0028a0900, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002d65780 sp=0xc002d65690 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0027cd110, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc00310e7e0}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002d65838 sp=0xc002d65780 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002799860, {0x43f1c20?, 0xc00310e7e0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002d65a98 sp=0xc002d65838 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0028a0654?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002d65af8 sp=0xc002d65a98 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc0028a05a0, 0xc002d65cb0, 0xc002d65c80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002d65ba0 sp=0xc002d65af8 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc0028a05a0, {0x43f1c20, 0xc00310e7e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002d65ce0 sp=0xc002d65ba0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0027222c0, {0x43f1c20, 0xc00310e7e0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc002d65ea8 sp=0xc002d65ce0 pc=0x12ac5b9 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectClient).Recv(0xc002857d80) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:71 +0x66 fp=0xc002d65ee8 sp=0xc002d65ea8 pc=0x2003b86 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002d65fe0 sp=0xc002d65ee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d65fe8 sp=0xc002d65fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1559 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1580 [select]: +runtime.gopark(0xc002e00fb0?, 0x2?, 0x0?, 0x0?, 0xc002e00f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e00da8 sp=0xc002e00d88 pc=0x4777ae +runtime.selectgo(0xc002e00fb0, 0xc002e00f28, 0x4b0571?, 0x0, 0x11847d3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e00ef8 sp=0xc002e00da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002e00fe0 sp=0xc002e00ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e00fe8 sp=0xc002e00fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1581 [select]: +runtime.gopark(0xc002e01f90?, 0x2?, 0xff?, 0xff?, 0xc002e01eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e01d28 sp=0xc002e01d08 pc=0x4777ae +runtime.selectgo(0xc002e01f90, 0xc002e01ea8, 0x4b0571?, 0x0, 0x448855?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e01e78 sp=0xc002e01d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc000f25d40) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002e01fc0 sp=0xc002e01e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002e01fe0 sp=0xc002e01fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e01fe8 sp=0xc002e01fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1582 [select]: +runtime.gopark(0xc002e3ff40?, 0x2?, 0xb4?, 0x1b?, 0xc002e3fecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e3fd48 sp=0xc002e3fd28 pc=0x4777ae +runtime.selectgo(0xc002e3ff40, 0xc002e3fec8, 0x4ad421?, 0x0, 0x1381849?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e3fe98 sp=0xc002e3fd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002e14690}, 0xc000f25dd0, 0x0, 0xc002e0ad20, 0xc002c7f1b0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002e3ff70 sp=0xc002e3fe98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002e3ffe0 sp=0xc002e3ff70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e3ffe8 sp=0xc002e3ffe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1583 [chan receive]: +runtime.gopark(0xc0027cd320?, 0x4b05a9?, 0x50?, 0x3e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e03e20 sp=0xc002e03e00 pc=0x4777ae +runtime.chanrecv(0xc00235d800, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e03e98 sp=0xc002e03e20 pc=0x441325 +runtime.chanrecv1(0xc002c73ce0?, 0x2005f24?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e03ec0 sp=0xc002e03e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e03fe0 sp=0xc002e03ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e03fe8 sp=0xc002e03fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1584 [chan receive]: +runtime.gopark(0xc0027cd320?, 0x4b05a9?, 0x50?, 0xbe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cdbe20 sp=0xc002cdbe00 pc=0x4777ae +runtime.chanrecv(0xc00235d800, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cdbe98 sp=0xc002cdbe20 pc=0x441325 +runtime.chanrecv1(0xc002c73ce0?, 0x6a55575255575268?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cdbec0 sp=0xc002cdbe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cdbfe0 sp=0xc002cdbec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cdbfe8 sp=0xc002cdbfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1585 [chan receive]: +runtime.gopark(0xc0027cd320?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e04e20 sp=0xc002e04e00 pc=0x4777ae +runtime.chanrecv(0xc00235d800, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e04e98 sp=0xc002e04e20 pc=0x441325 +runtime.chanrecv1(0xc002c73ce0?, 0x4166b20?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e04ec0 sp=0xc002e04e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e04fe0 sp=0xc002e04ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e04fe8 sp=0xc002e04fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1602 [chan receive]: +runtime.gopark(0xc0027cd320?, 0x4b05a9?, 0x50?, 0x7e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e07e20 sp=0xc002e07e00 pc=0x4777ae +runtime.chanrecv(0xc00235d800, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e07e98 sp=0xc002e07e20 pc=0x441325 +runtime.chanrecv1(0xc002c73ce0?, 0x1102359?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e07ec0 sp=0xc002e07e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e07fe0 sp=0xc002e07ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e07fe8 sp=0xc002e07fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1603 [select]: +runtime.gopark(0xc002e1baa8?, 0x2?, 0x67?, 0x6a?, 0xc002e1b9c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e1b838 sp=0xc002e1b818 pc=0x4777ae +runtime.selectgo(0xc002e1baa8, 0xc002e1b9c0, 0xc002ccea20?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e1b988 sp=0xc002e1b838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc002c2ad20) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002e1bad8 sp=0xc002e1b988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc00285a120, {0x5b426a8, 0xc002c73ce0}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002e1bcf8 sp=0xc002e1bad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc002e1bfe0 sp=0xc002e1bcf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e1bfe8 sp=0xc002e1bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 1546 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 1561 [select]: +runtime.gopark(0xc002e1d710?, 0x2?, 0x0?, 0x0?, 0xc002e1d6bc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e1d528 sp=0xc002e1d508 pc=0x4777ae +runtime.selectgo(0xc002e1d710, 0xc002e1d6b8, 0x7f429a4a5300?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e1d678 sp=0xc002e1d528 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0027caeb0, {0xc002c73b40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002e1d740 sp=0xc002e1d678 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0027caeb0, {0xc002c73b40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002e1d7d8 sp=0xc002e1d740 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0027cd350, {0xc002c73b40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002e1d820 sp=0xc002e1d7d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0027cd350}, {0xc002c73b40, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002e1d868 sp=0xc002e1d820 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0028a0c60, {0xc002c73b40, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002e1d8d8 sp=0xc002e1d868 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c73b30, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002e1d9a8 sp=0xc002e1d8d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc0028a0c60, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002e1da98 sp=0xc002e1d9a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c73b30, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc0027acfc0}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002e1db50 sp=0xc002e1da98 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc0016710e0, {0x43f1c20?, 0xc0027acfc0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002e1dd88 sp=0xc002e1db50 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002c73c80, {0x43f1c20, 0xc0027acfc0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002e1de70 sp=0xc002e1dd88 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002bf9680, {0x43f1c20, 0xc0027acfc0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002e1dea8 sp=0xc002e1de70 pc=0x1fd94b2 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectServer).Recv(0xc002c7f110) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:140 +0x66 fp=0xc002e1dee8 sp=0xc002e1dea8 pc=0x2004046 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002e1dfe0 sp=0xc002e1dee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e1dfe8 sp=0xc002e1dfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1603 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1604 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc00235d560?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db6e20 sp=0xc002db6e00 pc=0x4777ae +runtime.chanrecv(0xc00235d560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002db6e98 sp=0xc002db6e20 pc=0x441325 +runtime.chanrecv1(0xc0027cd020?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002db6ec0 sp=0xc002db6e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002db6fe0 sp=0xc002db6ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db6fe8 sp=0xc002db6fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1605 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc00235d560?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cd7e20 sp=0xc002cd7e00 pc=0x4777ae +runtime.chanrecv(0xc00235d560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cd7e98 sp=0xc002cd7e20 pc=0x441325 +runtime.chanrecv1(0xc0027cd020?, 0x1102359?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cd7ec0 sp=0xc002cd7e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cd7fe0 sp=0xc002cd7ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cd7fe8 sp=0xc002cd7fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1606 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc00235d560?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e05e20 sp=0xc002e05e00 pc=0x4777ae +runtime.chanrecv(0xc00235d560, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e05e98 sp=0xc002e05e20 pc=0x441325 +runtime.chanrecv1(0xc0027cd020?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e05ec0 sp=0xc002e05e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e05fe0 sp=0xc002e05ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e05fe8 sp=0xc002e05fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1607 [select]: +runtime.gopark(0xc002cd8fb0?, 0x2?, 0xff?, 0xff?, 0xc002cd8f6c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cd8dd8 sp=0xc002cd8db8 pc=0x4777ae +runtime.selectgo(0xc002cd8fb0, 0xc002cd8f68, 0x202020200a353620?, 0x0, 0x20200a3936202d20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cd8f28 sp=0xc002cd8dd8 pc=0x48916b +context.(*cancelCtx).propagateCancel.func2() + /home/ec2-user/sdk/go1.21.0/src/context/context.go:505 +0xe5 fp=0xc002cd8fe0 sp=0xc002cd8f28 pc=0x5d5bc5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cd8fe8 sp=0xc002cd8fe0 pc=0x4ad421 +created by context.(*cancelCtx).propagateCancel in goroutine 1174 + /home/ec2-user/sdk/go1.21.0/src/context/context.go:504 +0x530 + +goroutine 1608 [select]: +runtime.gopark(0xc002e06fb0?, 0x2?, 0xb4?, 0x1b?, 0xc002e06f74?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e06de8 sp=0xc002e06dc8 pc=0x4777ae +runtime.selectgo(0xc002e06fb0, 0xc002e06f70, 0x21557a5?, 0x0, 0xc0026882c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e06f38 sp=0xc002e06de8 pc=0x48916b +google.golang.org/grpc.newClientStreamWithParams.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:383 +0xde fp=0xc002e06fe0 sp=0xc002e06f38 pc=0x118b45e +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e06fe8 sp=0xc002e06fe0 pc=0x4ad421 +created by google.golang.org/grpc.newClientStreamWithParams in goroutine 1174 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:382 +0x1b10 + +goroutine 1609 [select]: +runtime.gopark(0xc002cd9fb0?, 0x2?, 0x0?, 0x0?, 0xc002cd9ee4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cd9d58 sp=0xc002cd9d38 pc=0x4777ae +runtime.selectgo(0xc002cd9fb0, 0xc002cd9ee0, 0x3737202d20202020?, 0x0, 0x2d202020200a3430?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cd9ea8 sp=0xc002cd9d58 pc=0x48916b +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream.func1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:203 +0x169 fp=0xc002cd9fe0 sp=0xc002cd9ea8 pc=0x12ad6c9 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cd9fe8 sp=0xc002cd9fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.wrapClientStream in goroutine 1174 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:199 +0x1a8 + +goroutine 1610 [chan receive]: +runtime.gopark(0x4b0571?, 0x12a7458?, 0xd0?, 0x2d?, 0x1377450?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e02d60 sp=0xc002e02d40 pc=0x4777ae +runtime.chanrecv(0xc002e18420, 0xc002e02e60, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e02dd8 sp=0xc002e02d60 pc=0x441325 +runtime.chanrecv1(0x12a7405?, 0xc002e02e38?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e02e00 sp=0xc002e02dd8 pc=0x440f72 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1.1() + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:283 +0x78 fp=0xc002e02fe0 sp=0xc002e02e00 pc=0x12ae3d8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e02fe8 sp=0xc002e02fe0 pc=0x4ad421 +created by go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamClientInterceptor.func1 in goroutine 1174 + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:282 +0x7c5 + +goroutine 1587 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ce8d98 sp=0xc002ce8d78 pc=0x4777ae +runtime.chanrecv(0xc002ccf740, 0xc002ce8fb0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002ce8e10 sp=0xc002ce8d98 pc=0x441325 +runtime.chanrecv1(0x0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002ce8e38 sp=0xc002ce8e10 pc=0x440f72 +github.com/rancher/opni/pkg/plugins/apis/apiextensions/stream.(*gatewayStreamExtensionServerImpl).Connect(0xc0007d1b90, {0x5b508a0, 0xc002cab6d0}) + /home/ec2-user/opni/pkg/plugins/apis/apiextensions/stream/plugin_gateway.go:157 +0xe51 fp=0xc002ce9370 sp=0xc002ce8e38 pc=0x261f931 +github.com/rancher/opni/pkg/apis/stream/v1._Stream_Connect_Handler({0x4229080?, 0xc0007d1b90}, {0x5b4a6f0?, 0xc002cc21e0}) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:121 +0xb6 fp=0xc002ce93b0 sp=0xc002ce9370 pc=0x2003ed6 +github.com/rancher/opni/pkg/plugins.ServeConfig.func1.1({0x4229080, 0xc0007d1b90}, {0x5b4ab30, 0xc002cd24b0}, 0xc00217f5f0?, 0x47ee1d0) + /home/ec2-user/opni/pkg/plugins/client.go:104 +0x1ed fp=0xc002ce9400 sp=0xc002ce93b0 pc=0x218992d +google.golang.org/grpc.getChainStreamHandler.func1({0x4229080, 0xc0007d1b90}, {0x5b4ab30, 0xc002cd24b0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1506 +0x115 fp=0xc002ce9490 sp=0xc002ce9400 pc=0x117fe75 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1({0x4229080, 0xc0007d1b90}, {0x5b492f8, 0xc001389c20}, 0xc00217f5f0, 0xc002571f40) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:437 +0x602 fp=0xc002ce97d0 sp=0xc002ce9490 pc=0x12b0e02 +google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x4229080, 0xc0007d1b90}, {0x5b492f8, 0xc001389c20}, 0x3f252a0?, 0xc002cab5b0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1497 +0xcf fp=0xc002ce9848 sp=0xc002ce97d0 pc=0x1175a4f +google.golang.org/grpc.(*Server).processStreamingRPC(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc002906240, 0xc0010e4f90, 0x770c800, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1661 +0x20b1 fp=0xc002ce9db0 sp=0xc002ce9848 pc=0x1181f91 +google.golang.org/grpc.(*Server).handleStream(0xc00191c5a0, {0x5b54e00, 0xc00183b520}, 0xc002906240, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:1741 +0xf65 fp=0xc002ce9f28 sp=0xc002ce9db0 pc=0x1184745 +google.golang.org/grpc.(*Server).serveStreams.func1.1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:982 +0xe7 fp=0xc002ce9fe0 sp=0xc002ce9f28 pc=0x11794a7 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ce9fe8 sp=0xc002ce9fe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 703 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:980 +0x2c6 + +goroutine 1611 [select]: +runtime.gopark(0xc002db2fb0?, 0x2?, 0x0?, 0x2e?, 0xc002db2f2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db2da8 sp=0xc002db2d88 pc=0x4777ae +runtime.selectgo(0xc002db2fb0, 0xc002db2f28, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002db2ef8 sp=0xc002db2da8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002db2fe0 sp=0xc002db2ef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db2fe8 sp=0xc002db2fe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1612 [select]: +runtime.gopark(0xc002db3f90?, 0x2?, 0xff?, 0xff?, 0xc002db3eac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db3d28 sp=0xc002db3d08 pc=0x4777ae +runtime.selectgo(0xc002db3f90, 0xc002db3ea8, 0x3736202d20202020?, 0x0, 0x202d202020200a30?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002db3e78 sp=0xc002db3d28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc0011ea240) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002db3fc0 sp=0xc002db3e78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002db3fe0 sp=0xc002db3fc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db3fe8 sp=0xc002db3fe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1613 [select]: +runtime.gopark(0xc002e59f40?, 0x2?, 0xb4?, 0x1b?, 0xc002e59ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e59d48 sp=0xc002e59d28 pc=0x4777ae +runtime.selectgo(0xc002e59f40, 0xc002e59ec8, 0x4ad421?, 0x0, 0x1381849?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e59e98 sp=0xc002e59d48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002e14f50}, 0xc0011ea2d0, 0x0, 0xc002e0b9e0, 0xc002c7f940) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc002e59f70 sp=0xc002e59e98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc002e59fe0 sp=0xc002e59f70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e59fe8 sp=0xc002e59fe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1614 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc002858a80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002eb8e20 sp=0xc002eb8e00 pc=0x4777ae +runtime.chanrecv(0xc002858a80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002eb8e98 sp=0xc002eb8e20 pc=0x441325 +runtime.chanrecv1(0xc002e35050?, 0x1102359?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002eb8ec0 sp=0xc002eb8e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002eb8fe0 sp=0xc002eb8ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002eb8fe8 sp=0xc002eb8fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1615 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc002858a80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002eb9e20 sp=0xc002eb9e00 pc=0x4777ae +runtime.chanrecv(0xc002858a80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002eb9e98 sp=0xc002eb9e20 pc=0x441325 +runtime.chanrecv1(0xc002e35050?, 0xc000f3e6e8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002eb9ec0 sp=0xc002eb9e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002eb9fe0 sp=0xc002eb9ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002eb9fe8 sp=0xc002eb9fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1616 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x7e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002eb7e20 sp=0xc002eb7e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002eb7e98 sp=0xc002eb7e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x6a55575255575268?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002eb7ec0 sp=0xc002eb7e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002eb7fe0 sp=0xc002eb7ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002eb7fe8 sp=0xc002eb7fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1617 [select]: +runtime.gopark(0xc002ea1c98?, 0x2?, 0x67?, 0x6a?, 0xc002ea1bb4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ea1a28 sp=0xc002ea1a08 pc=0x4777ae +runtime.selectgo(0xc002ea1c98, 0xc002ea1bb0, 0xc002ea1bc0?, 0x0, 0x43f1c20?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ea1b78 sp=0xc002ea1a28 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc002c2adc0) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002ea1cc8 sp=0xc002ea1b78 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc00285aea0, {0x5b426a8, 0xc002e35050}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002ea1ee8 sp=0xc002ea1cc8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Splice.func2() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:190 +0x6b fp=0xc002ea1fe0 sp=0xc002ea1ee8 pc=0x136faab +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ea1fe8 sp=0xc002ea1fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Splice in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:189 +0x61a + +goroutine 1588 [select]: +runtime.gopark(0xc002cfffb0?, 0x2?, 0x0?, 0x0?, 0xc002cfff2c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cffda8 sp=0xc002cffd88 pc=0x4777ae +runtime.selectgo(0xc002cfffb0, 0xc002cfff28, 0x4b0571?, 0x0, 0x11847d3?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cffef8 sp=0xc002cffda8 pc=0x48916b +github.com/kralicky/totem.newRecvWrapper.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:105 +0xcc fp=0xc002cfffe0 sp=0xc002cffef8 pc=0x136decc +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cfffe8 sp=0xc002cfffe0 pc=0x4ad421 +created by github.com/kralicky/totem.newRecvWrapper in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:104 +0x2ea + +goroutine 1589 [select]: +runtime.gopark(0xc002cfdf90?, 0x2?, 0xb4?, 0x1b?, 0xc002cfdeac?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cfdd28 sp=0xc002cfdd08 pc=0x4777ae +runtime.selectgo(0xc002cfdf90, 0xc002cfdea8, 0x5d53d0?, 0x0, 0xc002669810?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002cfde78 sp=0xc002cfdd28 pc=0x48916b +github.com/alitto/pond.(*WorkerPool).purge(0xc002949c80) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:384 +0x1d7 fp=0xc002cfdfc0 sp=0xc002cfde78 pc=0x12a6ed7 +github.com/alitto/pond.New.func2() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x34 fp=0xc002cfdfe0 sp=0xc002cfdfc0 pc=0x12a58b4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cfdfe8 sp=0xc002cfdfe0 pc=0x4ad421 +created by github.com/alitto/pond.New in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:144 +0x4a5 + +goroutine 1590 [select]: +runtime.gopark(0xc00301bf40?, 0x2?, 0xb4?, 0x1b?, 0xc00301becc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00301bd48 sp=0xc00301bd28 pc=0x4777ae +runtime.selectgo(0xc00301bf40, 0xc00301bec8, 0x5d8cc5?, 0x0, 0xc002d1c990?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00301be98 sp=0xc00301bd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002cba780}, 0xc002949d10, 0x0, 0xc002ccef60, 0xc002cab960) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc00301bf70 sp=0xc00301be98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc00301bfe0 sp=0xc00301bf70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00301bfe8 sp=0xc00301bfe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1591 [chan receive]: +runtime.gopark(0xc002cd22a0?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cd5e20 sp=0xc002cd5e00 pc=0x4777ae +runtime.chanrecv(0xc002858ae0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cd5e98 sp=0xc002cd5e20 pc=0x441325 +runtime.chanrecv1(0xc002cd2510?, 0x448845?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cd5ec0 sp=0xc002cd5e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cd5fe0 sp=0xc002cd5ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cd5fe8 sp=0xc002cd5fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1592 [chan receive]: +runtime.gopark(0xc002cd22a0?, 0x4b05a9?, 0x50?, 0x6e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cd6e20 sp=0xc002cd6e00 pc=0x4777ae +runtime.chanrecv(0xc002858ae0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cd6e98 sp=0xc002cd6e20 pc=0x441325 +runtime.chanrecv1(0xc002cd2510?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cd6ec0 sp=0xc002cd6e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cd6fe0 sp=0xc002cd6ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cd6fe8 sp=0xc002cd6fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1593 [chan receive]: +runtime.gopark(0xc002cd22a0?, 0x4b05a9?, 0x50?, 0xbe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cfbe20 sp=0xc002cfbe00 pc=0x4777ae +runtime.chanrecv(0xc002858ae0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cfbe98 sp=0xc002cfbe20 pc=0x441325 +runtime.chanrecv1(0xc002cd2510?, 0x1174725?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cfbec0 sp=0xc002cfbe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cfbfe0 sp=0xc002cfbec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cfbfe8 sp=0xc002cfbfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1594 [chan receive]: +runtime.gopark(0xc002cd22a0?, 0x4b05a9?, 0x50?, 0xce?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cfce20 sp=0xc002cfce00 pc=0x4777ae +runtime.chanrecv(0xc002858ae0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cfce98 sp=0xc002cfce20 pc=0x441325 +runtime.chanrecv1(0xc002cd2510?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cfcec0 sp=0xc002cfce98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cfcfe0 sp=0xc002cfcec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cfcfe8 sp=0xc002cfcfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1595 [chan receive]: +runtime.gopark(0xc002cd22a0?, 0x4b05a9?, 0x50?, 0xae?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cdae20 sp=0xc002cdae00 pc=0x4777ae +runtime.chanrecv(0xc002858ae0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cdae98 sp=0xc002cdae20 pc=0x441325 +runtime.chanrecv1(0xc002cd2510?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cdaec0 sp=0xc002cdae98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cdafe0 sp=0xc002cdaec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cdafe8 sp=0xc002cdafe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1596 [select]: +runtime.gopark(0xc002d2daa8?, 0x2?, 0x67?, 0x6a?, 0xc002d2d9c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d2d838 sp=0xc002d2d818 pc=0x4777ae +runtime.selectgo(0xc002d2daa8, 0xc002d2d9c0, 0xc002ccf980?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d2d988 sp=0xc002d2d838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc0024e94a0) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002d2dad8 sp=0xc002d2d988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc002907680, {0x5b426a8, 0xc002cd2510}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002d2dcf8 sp=0xc002d2dad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc002d2dfe0 sp=0xc002d2dcf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d2dfe8 sp=0xc002d2dfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 1587 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 1597 [select]: +runtime.gopark(0xc002d2f710?, 0x2?, 0x0?, 0x0?, 0xc002d2f6bc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d2f528 sp=0xc002d2f508 pc=0x4777ae +runtime.selectgo(0xc002d2f710, 0xc002d2f6b8, 0x7f429a4a8600?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d2f678 sp=0xc002d2f528 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002cba460, {0xc002cd2370, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002d2f740 sp=0xc002d2f678 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002cba460, {0xc002cd2370, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002d2f7d8 sp=0xc002d2f740 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002cd22d0, {0xc002cd2370, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002d2f820 sp=0xc002d2f7d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002cd22d0}, {0xc002cd2370, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002d2f868 sp=0xc002d2f820 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc002906240, {0xc002cd2370, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002d2f8d8 sp=0xc002d2f868 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002cd2360, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002d2f9a8 sp=0xc002d2f8d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc002906240, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002d2fa98 sp=0xc002d2f9a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002cd2360, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc002e99d50}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002d2fb50 sp=0xc002d2fa98 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001389c20, {0x43f1c20?, 0xc002e99d50}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002d2fd88 sp=0xc002d2fb50 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002cd24b0, {0x43f1c20, 0xc002e99d50}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002d2fe70 sp=0xc002d2fd88 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002cc21e0, {0x43f1c20, 0xc002e99d50}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002d2fea8 sp=0xc002d2fe70 pc=0x1fd94b2 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectServer).Recv(0xc002cab6d0) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:140 +0x66 fp=0xc002d2fee8 sp=0xc002d2fea8 pc=0x2004046 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002d2ffe0 sp=0xc002d2fee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d2ffe8 sp=0xc002d2ffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1596 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1598 [select]: +runtime.gopark(0xc002d5f3f8?, 0x2?, 0xc0?, 0x72?, 0xc002d5f37c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d5f1e8 sp=0xc002d5f1c8 pc=0x4777ae +runtime.selectgo(0xc002d5f3f8, 0xc002d5f378, 0x5b203e8?, 0x0, 0x2710?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d5f338 sp=0xc002d5f1e8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002e14e60, {0xc002e35150, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002d5f428 sp=0xc002d5f338 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002e14e60, {0xc002e35150, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002d5f4c0 sp=0xc002d5f428 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002e35110, {0xc002e35150, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002d5f508 sp=0xc002d5f4c0 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002e35110}, {0xc002e35150, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002d5f550 sp=0xc002d5f508 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00285ad80, {0xc002e35150, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002d5f5c0 sp=0xc002d5f550 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002e35140, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002d5f690 sp=0xc002d5f5c0 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc00285ad80, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002d5f780 sp=0xc002d5f690 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002e35140, {0x7f424347ae80, 0x779c980}, 0x5b16300?, {0x0, 0x0}, {0x43f1c20, 0xc002e6d8f0}, 0x5?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002d5f838 sp=0xc002d5f780 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002e2a680, {0x43f1c20?, 0xc002e6d8f0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002d5fa98 sp=0xc002d5f838 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00285aad4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002d5faf8 sp=0xc002d5fa98 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc00285aa20, 0xc002d5fcb0, 0xc002d5fc80) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002d5fba0 sp=0xc002d5faf8 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc00285aa20, {0x43f1c20, 0xc002e6d8f0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002d5fce0 sp=0xc002d5fba0 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc0027a7900, {0x43f1c20, 0xc002e6d8f0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc002d5fea8 sp=0xc002d5fce0 pc=0x12ac5b9 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectClient).Recv(0xc002c7f8d0) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:71 +0x66 fp=0xc002d5fee8 sp=0xc002d5fea8 pc=0x2003b86 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002d5ffe0 sp=0xc002d5fee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d5ffe8 sp=0xc002d5ffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1617 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1599 [chan receive]: +runtime.gopark(0x70?, 0x0?, 0x70?, 0x0?, 0xc002858a80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cfee20 sp=0xc002cfee00 pc=0x4777ae +runtime.chanrecv(0xc002858a80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cfee98 sp=0xc002cfee20 pc=0x441325 +runtime.chanrecv1(0xc002e35050?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cfeec0 sp=0xc002cfee98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cfefe0 sp=0xc002cfeec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cfefe8 sp=0xc002cfefe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1600 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc002858a80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ebbe20 sp=0xc002ebbe00 pc=0x4777ae +runtime.chanrecv(0xc002858a80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002ebbe98 sp=0xc002ebbe20 pc=0x441325 +runtime.chanrecv1(0xc002e35050?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002ebbec0 sp=0xc002ebbe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002ebbfe0 sp=0xc002ebbec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ebbfe8 sp=0xc002ebbfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1601 [chan receive]: +runtime.gopark(0x70?, 0x0?, 0x70?, 0x0?, 0xc002858a80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cf9e20 sp=0xc002cf9e00 pc=0x4777ae +runtime.chanrecv(0xc002858a80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cf9e98 sp=0xc002cf9e20 pc=0x441325 +runtime.chanrecv1(0xc002e35050?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cf9ec0 sp=0xc002cf9e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cf9fe0 sp=0xc002cf9ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cf9fe8 sp=0xc002cf9fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1618 [chan receive]: +runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0xc002858a80?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cf8e20 sp=0xc002cf8e00 pc=0x4777ae +runtime.chanrecv(0xc002858a80, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cf8e98 sp=0xc002cf8e20 pc=0x441325 +runtime.chanrecv1(0xc002e35050?, 0x2005f24?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cf8ec0 sp=0xc002cf8e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cf8fe0 sp=0xc002cf8ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cf8fe8 sp=0xc002cf8fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1619 [select]: +runtime.gopark(0xc002e9daa8?, 0x2?, 0x67?, 0x6a?, 0xc002e9d9c4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e9d838 sp=0xc002e9d818 pc=0x4777ae +runtime.selectgo(0xc002e9daa8, 0xc002e9d9c0, 0xc002047b60?, 0x0, 0x4aead9?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e9d988 sp=0xc002e9d838 pc=0x48916b +github.com/kralicky/totem.(*recvWrapper).Recv(0xc0025ca8c0) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:76 +0x1e9 fp=0xc002e9dad8 sp=0xc002e9d988 pc=0x136d589 +github.com/kralicky/totem.(*StreamController).Run(0xc0019cb8c0, {0x5b426a8, 0xc0027cc1b0}) + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/stream.go:299 +0x170 fp=0xc002e9dcf8 sp=0xc002e9dad8 pc=0x13766b0 +github.com/kralicky/totem.(*Server).Serve.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:258 +0xc6 fp=0xc002e9dfe0 sp=0xc002e9dcf8 pc=0x1371c26 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e9dfe8 sp=0xc002e9dfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*Server).Serve in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/server.go:257 +0x1c5 + +goroutine 1620 [select]: +runtime.gopark(0xc002e9f6d8?, 0x2?, 0x0?, 0x0?, 0xc002e9f684?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e9f4f0 sp=0xc002e9f4d0 pc=0x4777ae +runtime.selectgo(0xc002e9f6d8, 0xc002e9f680, 0x7f429a4a7100?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e9f640 sp=0xc002e9f4f0 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002c2e870, {0xc002c30a00, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002e9f708 sp=0xc002e9f640 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002c2e870, {0xc002c30a00, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002e9f7a0 sp=0xc002e9f708 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c30990, {0xc002c30a00, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002e9f7e8 sp=0xc002e9f7a0 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c30990}, {0xc002c30a00, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002e9f830 sp=0xc002e9f7e8 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8ad80, {0xc002c30a00, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002e9f8a0 sp=0xc002e9f830 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c309f0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002e9f970 sp=0xc002e9f8a0 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001d8ad80, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002e9fa60 sp=0xc002e9f970 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c309f0, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x43f1c20, 0xc002fe0460}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002e9fb18 sp=0xc002e9fa60 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001670d20, {0x43f1c20?, 0xc002fe0460}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002e9fd50 sp=0xc002e9fb18 pc=0x119905e +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002bf8da0, {0x43f1c20, 0xc002fe0460}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002e9fd88 sp=0xc002e9fd50 pc=0x1fd94b2 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002488960, {0x43f1c20, 0xc002fe0460}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002e9fdc0 sp=0xc002e9fd88 pc=0x1fd94b2 +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0027cc1e0, {0x43f1c20, 0xc002fe0460}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002e9fea8 sp=0xc002e9fdc0 pc=0x12b01b3 +github.com/rancher/opni/pkg/apis/stream/v1.(*streamConnectServer).Recv(0xc002857580) + /home/ec2-user/opni/pkg/apis/stream/v1/stream_grpc.pb.go:140 +0x66 fp=0xc002e9fee8 sp=0xc002e9fea8 pc=0x2004046 +github.com/kralicky/totem.(*recvWrapper).Start.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:53 +0x123 fp=0xc002e9ffe0 sp=0xc002e9fee8 pc=0x136d123 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e9ffe8 sp=0xc002e9ffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*recvWrapper).Start in goroutine 1619 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/recv.go:49 +0xe5 + +goroutine 1634 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x7e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e77e20 sp=0xc002e77e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e77e98 sp=0xc002e77e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e77ec0 sp=0xc002e77e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e77fe0 sp=0xc002e77ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e77fe8 sp=0xc002e77fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1635 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002db4e20 sp=0xc002db4e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002db4e98 sp=0xc002db4e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x2005f24?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002db4ec0 sp=0xc002db4e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002db4fe0 sp=0xc002db4ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002db4fe8 sp=0xc002db4fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1636 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002eb4e20 sp=0xc002eb4e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002eb4e98 sp=0xc002eb4e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002eb4ec0 sp=0xc002eb4e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002eb4fe0 sp=0xc002eb4ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002eb4fe8 sp=0xc002eb4fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1637 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x6e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002eb6e20 sp=0xc002eb6e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002eb6e98 sp=0xc002eb6e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002eb6ec0 sp=0xc002eb6e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002eb6fe0 sp=0xc002eb6ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002eb6fe8 sp=0xc002eb6fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1638 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002eb5e20 sp=0xc002eb5e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002eb5e98 sp=0xc002eb5e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x2005f24?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002eb5ec0 sp=0xc002eb5e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002eb5fe0 sp=0xc002eb5ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002eb5fe8 sp=0xc002eb5fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1639 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x3e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e73e20 sp=0xc002e73e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e73e98 sp=0xc002e73e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x1102359?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e73ec0 sp=0xc002e73e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e73fe0 sp=0xc002e73ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e73fe8 sp=0xc002e73fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1640 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e75e20 sp=0xc002e75e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e75e98 sp=0xc002e75e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e75ec0 sp=0xc002e75e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e75fe0 sp=0xc002e75ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e75fe8 sp=0xc002e75fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1641 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e74e20 sp=0xc002e74e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e74e98 sp=0xc002e74e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x5b058a8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e74ec0 sp=0xc002e74e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e74fe0 sp=0xc002e74ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e74fe8 sp=0xc002e74fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1642 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0xfe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e7fe20 sp=0xc002e7fe00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e7fe98 sp=0xc002e7fe20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e7fec0 sp=0xc002e7fe98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e7ffe0 sp=0xc002e7fec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e7ffe8 sp=0xc002e7ffe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1643 [chan receive]: +runtime.gopark(0xc002c30a50?, 0x4b05a9?, 0x50?, 0x6e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e76e20 sp=0xc002e76e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7380, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e76e98 sp=0xc002e76e20 pc=0x441325 +runtime.chanrecv1(0xc0027cc1b0?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e76ec0 sp=0xc002e76e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e76fe0 sp=0xc002e76ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e76fe8 sp=0xc002e76fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1174 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1648 [sync.Cond.Wait]: +runtime.gopark(0x4b0a55?, 0xc002fa45b0?, 0xff?, 0xff?, 0x4b05a9?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0029dfdd8 sp=0xc0029dfdb8 pc=0x4777ae +runtime.goparkunlock(...) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:404 +sync.runtime_notifyListWait(0xc002f90090, 0x3) + /home/ec2-user/sdk/go1.21.0/src/runtime/sema.go:527 +0x159 fp=0xc0029dfe28 sp=0xc0029dfdd8 pc=0x4a9339 +sync.(*Cond).Wait(0xc002f90080) + /home/ec2-user/sdk/go1.21.0/src/sync/cond.go:70 +0x75 fp=0xc0029dfe58 sp=0xc0029dfe28 pc=0x4bbff5 +github.com/nats-io/nats%2ego.(*Conn).waitForMsgs(0xc001659100, 0xc002fa45b0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:2959 +0x1a5 fp=0xc0029dffb0 sp=0xc0029dfe58 pc=0x1f76285 +github.com/nats-io/nats%2ego.(*Conn).subscribeLocked.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x45 fp=0xc0029dffe0 sp=0xc0029dffb0 pc=0x1f80b85 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0029dffe8 sp=0xc0029dffe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*Conn).subscribeLocked in goroutine 1139 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/nats.go:4232 +0x7d0 + +goroutine 1645 [select]: +runtime.gopark(0xc002e55ee0?, 0x4?, 0x67?, 0x6a?, 0xc002e55ba0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e559b8 sp=0xc002e55998 pc=0x4777ae +runtime.selectgo(0xc002e55ee0, 0xc002e55b98, 0x0?, 0x0, 0x3f99540?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e55b08 sp=0xc002e559b8 pc=0x48916b +github.com/rancher/opni/pkg/health.(*Listener).HandleConnection(0xc001623400, {0x5b430b0, 0xc0027cce40}, {0x7f421ad8c958, 0xc002e61a40}) + /home/ec2-user/opni/pkg/health/listener.go:195 +0xd1c fp=0xc002e55f30 sp=0xc002e55b08 pc=0x206ca3c +github.com/rancher/opni/pkg/health.(*Listener).HandleAgentConnection(0xc000f1ef48?, {0x5b430b0, 0xc0027cce40}, {0x5b2bf70?, 0xc002e61a40}) + /home/ec2-user/opni/pkg/health/listener.go:257 +0x65 fp=0xc002e55f78 sp=0xc002e55f30 pc=0x206d6a5 +github.com/rancher/opni/pkg/gateway.(*multiConnectionHandler).HandleAgentConnection.func1() + /home/ec2-user/opni/pkg/gateway/grpc.go:40 +0x82 fp=0xc002e55fe0 sp=0xc002e55f78 pc=0x22b3a62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e55fe8 sp=0xc002e55fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.(*multiConnectionHandler).HandleAgentConnection in goroutine 1644 + /home/ec2-user/opni/pkg/gateway/grpc.go:40 +0x6f + +goroutine 1646 [select]: +runtime.gopark(0xc002ebfe58?, 0x2?, 0x67?, 0x6a?, 0xc002ebfd94?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ebfbe8 sp=0xc002ebfbc8 pc=0x4777ae +runtime.selectgo(0xc002ebfe58, 0xc002ebfd90, 0xc002e4fa80?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002ebfd38 sp=0xc002ebfbe8 pc=0x48916b +github.com/rancher/opni/pkg/gateway.(*SyncRequester).runPeriodicSync(0xc0017e1740, {0x5b430b0?, 0xc0027cce40}, 0xc001d6fc90?, 0x45d964b800, 0x0?) + /home/ec2-user/opni/pkg/gateway/sync.go:110 +0x1b2 fp=0xc002ebfe88 sp=0xc002ebfd38 pc=0x22c1472 +github.com/rancher/opni/pkg/gateway.(*SyncRequester).HandleAgentConnection(0xc0017e1740, {0x5b430b0, 0xc0027cce40}, {0x5b2bf70?, 0xc002e61a40}) + /home/ec2-user/opni/pkg/gateway/sync.go:54 +0x3a5 fp=0xc002ebff78 sp=0xc002ebfe88 pc=0x22c0325 +github.com/rancher/opni/pkg/gateway.(*multiConnectionHandler).HandleAgentConnection.func1() + /home/ec2-user/opni/pkg/gateway/grpc.go:40 +0x82 fp=0xc002ebffe0 sp=0xc002ebff78 pc=0x22b3a62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ebffe8 sp=0xc002ebffe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.(*multiConnectionHandler).HandleAgentConnection in goroutine 1644 + /home/ec2-user/opni/pkg/gateway/grpc.go:40 +0x6f + +goroutine 1647 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0xb0?, 0x40?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002ee5cd8 sp=0xc002ee5cb8 pc=0x4777ae +runtime.chanrecv(0xc00235d3e0, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002ee5d50 sp=0xc002ee5cd8 pc=0x441325 +runtime.chanrecv1(0xc0027cce40?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002ee5d78 sp=0xc002ee5d50 pc=0x440f72 +github.com/rancher/opni/pkg/gateway.(*DelegateServer).HandleAgentConnection(0xc000ee40a0, {0x5b430b0, 0xc0027cce40}, {0x5b2bf70, 0xc002e61a40}) + /home/ec2-user/opni/pkg/gateway/delegate.go:74 +0x639 fp=0xc002ee5f78 sp=0xc002ee5d78 pc=0x22a7cd9 +github.com/rancher/opni/pkg/gateway.(*multiConnectionHandler).HandleAgentConnection.func1() + /home/ec2-user/opni/pkg/gateway/grpc.go:40 +0x82 fp=0xc002ee5fe0 sp=0xc002ee5f78 pc=0x22b3a62 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002ee5fe8 sp=0xc002ee5fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/gateway.(*multiConnectionHandler).HandleAgentConnection in goroutine 1644 + /home/ec2-user/opni/pkg/gateway/grpc.go:40 +0x6f + +goroutine 1650 [select]: +runtime.gopark(0xc002f85410?, 0x2?, 0x0?, 0x0?, 0xc002f85394?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002f85200 sp=0xc002f851e0 pc=0x4777ae +runtime.selectgo(0xc002f85410, 0xc002f85390, 0xc00097fb40?, 0x0, 0x44e7c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002f85350 sp=0xc002f85200 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002c2f810, {0xc002c31d80, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002f85440 sp=0xc002f85350 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002c2f810, {0xc002c31d80, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002f854d8 sp=0xc002f85440 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c31d40, {0xc002c31d80, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002f85520 sp=0xc002f854d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c31d40}, {0xc002c31d80, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002f85568 sp=0xc002f85520 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8b320, {0xc002c31d80, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002f855d8 sp=0xc002f85568 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c31d70, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002f856a8 sp=0xc002f855d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc002f857b0?, 0xc001d8b320, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002f85798 sp=0xc002f856a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c31d70, {0x7f424347ae80, 0x779c980}, 0x5b18680?, {0x0, 0x0}, {0x4592b00, 0xc002f8af60}, 0xc00?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002f85850 sp=0xc002f85798 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002c29790, {0x4592b00?, 0xc002f8af60}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002f85ab0 sp=0xc002f85850 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001d8b194?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002f85b10 sp=0xc002f85ab0 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001d8b0e0, 0xc002f85cc8, 0xc002f85c98) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002f85bb8 sp=0xc002f85b10 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8b0e0, {0x4592b00, 0xc002f8af60}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002f85cf8 sp=0xc002f85bb8 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc002852140, {0x4592b00, 0xc002f8af60}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc002f85ec0 sp=0xc002f85cf8 pc=0x12ac5b9 +github.com/rancher/opni/pkg/management.NewServer.unknownServiceHandler.func3.1() + /home/ec2-user/opni/pkg/management/extensions.go:538 +0x8c fp=0xc002f85f38 sp=0xc002f85ec0 pc=0x231dbec +golang.org/x/sync/errgroup.(*Group).Go.func1() + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:75 +0x77 fp=0xc002f85fe0 sp=0xc002f85f38 pc=0x2007477 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002f85fe8 sp=0xc002f85fe0 pc=0x4ad421 +created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1486 + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:72 +0x125 + +goroutine 1698 [select]: +runtime.gopark(0xc002e9bf70?, 0x2?, 0xb4?, 0x1b?, 0xc002e9bebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e9bd20 sp=0xc002e9bd00 pc=0x4777ae +runtime.selectgo(0xc002e9bf70, 0xc002e9beb8, 0xc00215fe40?, 0x0, 0xc00215fef0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002e9be70 sp=0xc002e9bd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002cc3be0, {0x5b426e0, 0xc002ef5f90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc002e9bfa0 sp=0xc002e9be70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc002e9bfe0 sp=0xc002e9bfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e9bfe8 sp=0xc002e9bfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 609 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1714 [IO wait]: +runtime.gopark(0x440100?, 0xc00313f878?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00313f830 sp=0xc00313f810 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc00313f868 sp=0xc00313f830 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c43d0, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc00313f888 sp=0xc00313f868 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc003058020, 0xc002ff6000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc00313f8d0 sp=0xc00313f888 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc003058000, {0xc002ff6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc00313f9c0 sp=0xc00313f8d0 pc=0x50e5a5 +net.(*netFD).Read(0xc003058000, {0xc002ff6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc00313fa20 sp=0xc00313f9c0 pc=0x89342b +net.(*conn).Read(0xc002f9a458, {0xc002ff6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc00313faa8 sp=0xc00313fa20 pc=0x8b176d +net.(*TCPConn).Read(0xc002f9a458, {0xc002ff6000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc00313faf0 sp=0xc00313faa8 pc=0x8d72f4 +bufio.(*Reader).Read(0xc002fd8fc0, {0xc002686900, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc00313fbb0 sp=0xc00313faf0 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc002fd8fc0}, {0xc002686900, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc00313fbf8 sp=0xc00313fbb0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc002686900, 0x9, 0x9}, {0x5b0f0a0, 0xc002fd8fc0}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc00313fc98 sp=0xc00313fbf8 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc0026868c0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc00313fdd0 sp=0xc00313fc98 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc002028240, 0xa9aef4?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x29b fp=0xc00313ffb0 sp=0xc00313fdd0 pc=0x1123adb +google.golang.org/grpc/internal/transport.newHTTP2Client.func11() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x45 fp=0xc00313ffe0 sp=0xc00313ffb0 pc=0x11145c5 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00313ffe8 sp=0xc00313ffe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1699 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x3036 + +goroutine 1649 [chan receive]: +runtime.gopark(0x4aef8c?, 0x4b0a55?, 0xf0?, 0x1c?, 0x485e4f?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001b59e58 sp=0xc001b59e38 pc=0x4777ae +runtime.chanrecv(0xc000c9a580, 0xc001b59f10, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc001b59ed0 sp=0xc001b59e58 pc=0x441325 +runtime.chanrecv2(0xc002731cf0?, 0x44642c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:447 +0x12 fp=0xc001b59ef8 sp=0xc001b59ed0 pc=0x440f92 +github.com/nats-io/nats%2ego.(*js).resetPendingAcksOnReconnect(0xc002731ce0) + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:692 +0x9c fp=0xc001b59fc0 sp=0xc001b59ef8 pc=0x1f3cd9c +github.com/nats-io/nats%2ego.(*js).newAsyncReply.func1() + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:672 +0x34 fp=0xc001b59fe0 sp=0xc001b59fc0 pc=0x1f3ccd4 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001b59fe8 sp=0xc001b59fe0 pc=0x4ad421 +created by github.com/nats-io/nats%2ego.(*js).newAsyncReply in goroutine 1139 + /home/ec2-user/go/pkg/mod/github.com/nats-io/nats.go@v1.28.0/js.go:672 +0x88d + +goroutine 1548 [select]: +runtime.gopark(0xc002fb16a0?, 0x2?, 0x0?, 0x0?, 0xc002fb164c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002fb14b8 sp=0xc002fb1498 pc=0x4777ae +runtime.selectgo(0xc002fb16a0, 0xc002fb1648, 0x7f429a4abf00?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002fb1608 sp=0xc002fb14b8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0027cafa0, {0xc0027cd540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002fb16d0 sp=0xc002fb1608 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0027cafa0, {0xc0027cd540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002fb1768 sp=0xc002fb16d0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0027cd4a0, {0xc0027cd540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002fb17b0 sp=0xc002fb1768 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc0027cd4a0}, {0xc0027cd540, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002fb17f8 sp=0xc002fb17b0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0028a0fc0, {0xc0027cd540, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002fb1868 sp=0xc002fb17f8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc0027cd530, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002fb1938 sp=0xc002fb1868 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x0?, 0xc0028a0fc0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002fb1a28 sp=0xc002fb1938 pc=0x1171c5c +google.golang.org/grpc.recv(0xc0027cd530, {0x7f424347ae80, 0x779c980}, 0xc001d8b194?, {0x0, 0x0}, {0x435c620, 0xc002e5dbc0}, 0x4b057b?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002fb1ae0 sp=0xc002fb1a28 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc00278a2d0, {0x435c620?, 0xc002e5dbc0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002fb1d18 sp=0xc002fb1ae0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc0027cd680, {0x435c620, 0xc002e5dbc0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002fb1e00 sp=0xc002fb1d18 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002d9c200, {0x435c620, 0xc002e5dbc0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002fb1e38 sp=0xc002fb1e00 pc=0x1fd94b2 +github.com/rancher/opni/plugins/alerting/apis/alertops.(*configReconcilerSyncConfigServer).Recv(0xc002db02a0) + /home/ec2-user/opni/plugins/alerting/apis/alertops/alertops_grpc.pb.go:487 +0x66 fp=0xc002fb1e78 sp=0xc002fb1e38 pc=0x2610ca6 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).SyncConfig.func1() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/admin.go:282 +0x1b8 fp=0xc002fb1fa0 sp=0xc002fb1e78 pc=0x395c078 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc002fb1fe0 sp=0xc002fb1fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002fb1fe8 sp=0xc002fb1fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1547 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1651 [select]: +runtime.gopark(0xc002333738?, 0x2?, 0x0?, 0x0?, 0xc0023336e4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002333550 sp=0xc002333530 pc=0x4777ae +runtime.selectgo(0xc002333738, 0xc0023336e0, 0xc0023336e8?, 0x0, 0x2?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0023336a0 sp=0xc002333550 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002c2f590, {0xc002c31900, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002333768 sp=0xc0023336a0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002c2f590, {0xc002c31900, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002333800 sp=0xc002333768 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c31860, {0xc002c31900, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002333848 sp=0xc002333800 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c31860}, {0xc002c31900, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002333890 sp=0xc002333848 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8afc0, {0xc002c31900, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002333900 sp=0xc002333890 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c318f0, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc0023339d0 sp=0xc002333900 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc002f8aea0?, 0xc001d8afc0, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002333ac0 sp=0xc0023339d0 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c318f0, {0x7f424347ae80, 0x779c980}, 0xc002cd4ce8?, {0x0, 0x0}, {0x4592b00, 0xc002f8af90}, 0x96?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002333b78 sp=0xc002333ac0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001670e10, {0x4592b00?, 0xc002f8af90}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002333db0 sp=0xc002333b78 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002c31a40, {0x4592b00, 0xc002f8af90}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002333e98 sp=0xc002333db0 pc=0x12b01b3 +github.com/rancher/opni/pkg/management.NewServer.unknownServiceHandler.func3.2() + /home/ec2-user/opni/pkg/management/extensions.go:551 +0x9a fp=0xc002333f38 sp=0xc002333e98 pc=0x231da1a +golang.org/x/sync/errgroup.(*Group).Go.func1() + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:75 +0x77 fp=0xc002333fe0 sp=0xc002333f38 pc=0x2007477 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002333fe8 sp=0xc002333fe0 pc=0x4ad421 +created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1486 + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:72 +0x125 + +goroutine 1549 [select]: +runtime.gopark(0xc002c776a0?, 0x2?, 0x0?, 0x0?, 0xc002c7764c?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c774b8 sp=0xc002c77498 pc=0x4777ae +runtime.selectgo(0xc002c776a0, 0xc002c77648, 0x7f429a4ac200?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002c77608 sp=0xc002c774b8 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002c2fdb0, {0xc002c72730, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002c776d0 sp=0xc002c77608 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002c2fdb0, {0xc002c72730, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002c77768 sp=0xc002c776d0 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c72690, {0xc002c72730, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002c777b0 sp=0xc002c77768 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c72690}, {0xc002c72730, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002c777f8 sp=0xc002c777b0 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8bb00, {0xc002c72730, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002c77868 sp=0xc002c777f8 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c72720, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002c77938 sp=0xc002c77868 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x779c980?, 0xc001d8bb00, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002c77a28 sp=0xc002c77938 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c72720, {0x7f424347ae80, 0x779c980}, 0x0?, {0x0, 0x0}, {0x435c620, 0xc00309a4e0}, 0x0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002c77ae0 sp=0xc002c77a28 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001670ff0, {0x435c620?, 0xc00309a4e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002c77d18 sp=0xc002c77ae0 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002c72870, {0x435c620, 0xc00309a4e0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002c77e00 sp=0xc002c77d18 pc=0x12b01b3 +github.com/rancher/opni/pkg/util/streams.(*ServerStreamWithContext).RecvMsg(0xc002bf92c0, {0x435c620, 0xc00309a4e0}) + /home/ec2-user/opni/pkg/util/streams/streams.go:135 +0x52 fp=0xc002c77e38 sp=0xc002c77e00 pc=0x1fd94b2 +github.com/rancher/opni/plugins/alerting/apis/alertops.(*configReconcilerSyncConfigServer).Recv(0xc002c7e0c0) + /home/ec2-user/opni/plugins/alerting/apis/alertops/alertops_grpc.pb.go:487 +0x66 fp=0xc002c77e78 sp=0xc002c77e38 pc=0x2610ca6 +github.com/rancher/opni/plugins/alerting/pkg/alerting.(*Plugin).SyncConfig.func1() + /home/ec2-user/opni/plugins/alerting/pkg/alerting/admin.go:282 +0x1b8 fp=0xc002c77fa0 sp=0xc002c77e78 pc=0x395c078 +github.com/samber/lo.Async[...].func1() + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:32 +0x34 fp=0xc002c77fe0 sp=0xc002c77fa0 pc=0x176b214 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c77fe8 sp=0xc002c77fe0 pc=0x4ad421 +created by github.com/samber/lo.Async[...] in goroutine 1574 + /home/ec2-user/go/pkg/mod/github.com/samber/lo@v1.36.1-0.20230320154156-56ef8fe8a306/concurrency.go:31 +0x13d + +goroutine 1562 [select]: +runtime.gopark(0xc002d7f410?, 0x2?, 0x0?, 0x0?, 0xc002d7f394?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d7f200 sp=0xc002d7f1e0 pc=0x4777ae +runtime.selectgo(0xc002d7f410, 0xc002d7f390, 0xc00097fb40?, 0x0, 0x44e7c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d7f350 sp=0xc002d7f200 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc002c2fc70, {0xc002c724f0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:195 +0x136 fp=0xc002d7f440 sp=0xc002d7f350 pc=0x113bf56 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002c2fc70, {0xc002c724f0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:175 +0x35b fp=0xc002d7f4d8 sp=0xc002d7f440 pc=0x113ba5b +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c724b0, {0xc002c724f0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002d7f520 sp=0xc002d7f4d8 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c724b0}, {0xc002c724f0, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002d7f568 sp=0xc002d7f520 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8b9e0, {0xc002c724f0, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002d7f5d8 sp=0xc002d7f568 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c724e0, 0x7fffffff) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002d7f6a8 sp=0xc002d7f5d8 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0xc002d7f7b0?, 0xc001d8b9e0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002d7f798 sp=0xc002d7f6a8 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c724e0, {0x7f424347ae80, 0x779c980}, 0x5b18680?, {0x0, 0x0}, {0x4592b00, 0xc002d1def0}, 0xc00?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002d7f850 sp=0xc002d7f798 pc=0x11726e5 +google.golang.org/grpc.(*csAttempt).recvMsg(0xc002c29860, {0x4592b00?, 0xc002d1def0}, 0x0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1080 +0x548 fp=0xc002d7fab0 sp=0xc002d7f850 pc=0x1192568 +google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001d8b614?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x51 fp=0xc002d7fb10 sp=0xc002d7fab0 pc=0x1190ed1 +google.golang.org/grpc.(*clientStream).withRetry(0xc001d8b560, 0xc002d7fcc8, 0xc002d7fc98) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:752 +0x466 fp=0xc002d7fbb8 sp=0xc002d7fb10 pc=0x118ed46 +google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8b560, {0x4592b00, 0xc002d1def0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x17e fp=0xc002d7fcf8 sp=0xc002d7fbb8 pc=0x119077e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*clientStream).RecvMsg(0xc002852400, {0x4592b00, 0xc002d1def0}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:145 +0x79 fp=0xc002d7fec0 sp=0xc002d7fcf8 pc=0x12ac5b9 +github.com/rancher/opni/pkg/management.NewServer.unknownServiceHandler.func3.1() + /home/ec2-user/opni/pkg/management/extensions.go:538 +0x8c fp=0xc002d7ff38 sp=0xc002d7fec0 pc=0x231dbec +golang.org/x/sync/errgroup.(*Group).Go.func1() + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:75 +0x77 fp=0xc002d7ffe0 sp=0xc002d7ff38 pc=0x2007477 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d7ffe8 sp=0xc002d7ffe0 pc=0x4ad421 +created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1570 + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:72 +0x125 + +goroutine 1563 [select]: +runtime.gopark(0xc002d61738?, 0x2?, 0x20?, 0xdf?, 0xc002d616e4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002d61550 sp=0xc002d61530 pc=0x4777ae +runtime.selectgo(0xc002d61738, 0xc002d616e0, 0x7f429a4ab000?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002d616a0 sp=0xc002d61550 pc=0x48916b +google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc002c2f9f0, {0xc002c72070, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:183 +0x129 fp=0xc002d61768 sp=0xc002d616a0 pc=0x113bd29 +google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc002c2f9f0, {0xc002c72070, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:177 +0x3d5 fp=0xc002d61800 sp=0xc002d61768 pc=0x113bad5 +google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002c31fb0, {0xc002c72070, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:514 +0x5d fp=0xc002d61848 sp=0xc002d61800 pc=0x113dedd +io.ReadAtLeast({0x5b16300, 0xc002c31fb0}, {0xc002c72070, 0x5, 0x5}, 0x5) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002d61890 sp=0xc002d61848 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001d8b440, {0xc002c72070, 0x5, 0x5}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:498 +0x125 fp=0xc002d61900 sp=0xc002d61890 pc=0x113de05 +google.golang.org/grpc.(*parser).recvMsg(0xc002c72060, 0x400000) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:600 +0x6e fp=0xc002d619d0 sp=0xc002d61900 pc=0x1170fae +google.golang.org/grpc.recvAndDecompress(0x60?, 0xc001d8b440, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:730 +0x7c fp=0xc002d61ac0 sp=0xc002d619d0 pc=0x1171c5c +google.golang.org/grpc.recv(0xc002c72060, {0x7f424347ae80, 0x779c980}, 0xc002c2fae0?, {0x0, 0x0}, {0x4592b00, 0xc002f8b860}, 0xc002c723f0?, 0x0, ...) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/rpc_util.go:796 +0xc5 fp=0xc002d61b78 sp=0xc002d61ac0 pc=0x11726e5 +google.golang.org/grpc.(*serverStream).RecvMsg(0xc001670f00, {0x4592b00?, 0xc002f8b860}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1712 +0x2be fp=0xc002d61db0 sp=0xc002d61b78 pc=0x119905e +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc002c721b0, {0x4592b00, 0xc002f8b860}) + /home/ec2-user/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.42.0/interceptor.go:376 +0x73 fp=0xc002d61e98 sp=0xc002d61db0 pc=0x12b01b3 +github.com/rancher/opni/pkg/management.NewServer.unknownServiceHandler.func3.2() + /home/ec2-user/opni/pkg/management/extensions.go:551 +0x9a fp=0xc002d61f38 sp=0xc002d61e98 pc=0x231da1a +golang.org/x/sync/errgroup.(*Group).Go.func1() + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:75 +0x77 fp=0xc002d61fe0 sp=0xc002d61f38 pc=0x2007477 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002d61fe8 sp=0xc002d61fe0 pc=0x4ad421 +created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1570 + /home/ec2-user/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:72 +0x125 + +goroutine 1564 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0017a5e20 sp=0xc0017a5e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc0017a5e98 sp=0xc0017a5e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x1102359?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc0017a5ec0 sp=0xc0017a5e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc0017a5fe0 sp=0xc0017a5ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0017a5fe8 sp=0xc0017a5fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1565 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x4e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002cd4e20 sp=0xc002cd4e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002cd4e98 sp=0xc002cd4e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0xc000f3e6e8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002cd4ec0 sp=0xc002cd4e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002cd4fe0 sp=0xc002cd4ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002cd4fe8 sp=0xc002cd4fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1566 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x6e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c66e20 sp=0xc002c66e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c66e98 sp=0xc002c66e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x6a55575255575268?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c66ec0 sp=0xc002c66e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c66fe0 sp=0xc002c66ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c66fe8 sp=0xc002c66fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1567 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x7e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c67e20 sp=0xc002c67e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c67e98 sp=0xc002c67e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c67ec0 sp=0xc002c67e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c67fe0 sp=0xc002c67ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c67fe8 sp=0xc002c67fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1568 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x8e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002c68e20 sp=0xc002c68e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002c68e98 sp=0xc002c68e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002c68ec0 sp=0xc002c68e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002c68fe0 sp=0xc002c68ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002c68fe8 sp=0xc002c68fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1569 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0xe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e70e20 sp=0xc002e70e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e70e98 sp=0xc002e70e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0xc002009ee0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e70ec0 sp=0xc002e70e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e70fe0 sp=0xc002e70ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e70fe8 sp=0xc002e70fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1666 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x2e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e72e20 sp=0xc002e72e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e72e98 sp=0xc002e72e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x2106cfa?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e72ec0 sp=0xc002e72e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e72fe0 sp=0xc002e72ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e72fe8 sp=0xc002e72fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1667 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x1e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e71e20 sp=0xc002e71e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e71e98 sp=0xc002e71e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x2166b62?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e71ec0 sp=0xc002e71e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e71fe0 sp=0xc002e71ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e71fe8 sp=0xc002e71fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1668 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0xbe?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e7be20 sp=0xc002e7be00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e7be98 sp=0xc002e7be20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x4b0571?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e7bec0 sp=0xc002e7be98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e7bfe0 sp=0xc002e7bec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e7bfe8 sp=0xc002e7bfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1669 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0xce?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e7ce20 sp=0xc002e7ce00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e7ce98 sp=0xc002e7ce20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x448845?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e7cec0 sp=0xc002e7ce98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e7cfe0 sp=0xc002e7cec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e7cfe8 sp=0xc002e7cfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1670 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0xde?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e7de20 sp=0xc002e7de00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e7de98 sp=0xc002e7de20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x2106cfa?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e7dec0 sp=0xc002e7de98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e7dfe0 sp=0xc002e7dec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e7dfe8 sp=0xc002e7dfe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1671 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x5e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002fc5e20 sp=0xc002fc5e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002fc5e98 sp=0xc002fc5e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x5b50438?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002fc5ec0 sp=0xc002fc5e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002fc5fe0 sp=0xc002fc5ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002fc5fe8 sp=0xc002fc5fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1672 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0xee?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e7ee20 sp=0xc002e7ee00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e7ee98 sp=0xc002e7ee20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0x1102359?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e7eec0 sp=0xc002e7ee98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e7efe0 sp=0xc002e7eec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e7efe8 sp=0xc002e7efe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1673 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x8e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e78e20 sp=0xc002e78e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e78e98 sp=0xc002e78e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0xc000f3e6e8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e78ec0 sp=0xc002e78e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e78fe0 sp=0xc002e78ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e78fe8 sp=0xc002e78fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1674 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0xae?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e7ae20 sp=0xc002e7ae00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e7ae98 sp=0xc002e7ae20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e7aec0 sp=0xc002e7ae98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e7afe0 sp=0xc002e7aec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e7afe8 sp=0xc002e7afe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1675 [chan receive]: +runtime.gopark(0x4b0571?, 0x4b05a9?, 0x50?, 0x9e?, 0x5d8cae?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002e79e20 sp=0xc002e79e00 pc=0x4777ae +runtime.chanrecv(0xc002bf7320, 0x0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002e79e98 sp=0xc002e79e20 pc=0x441325 +runtime.chanrecv1(0xc002849020?, 0xc001150090?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002e79ec0 sp=0xc002e79e98 pc=0x440f72 +github.com/kralicky/totem.(*ServiceHandlerList).Append.func1() + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:111 +0x70 fp=0xc002e79fe0 sp=0xc002e79ec0 pc=0x1381850 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002e79fe8 sp=0xc002e79fe0 pc=0x4ad421 +created by github.com/kralicky/totem.(*ServiceHandlerList).Append in goroutine 1478 + /home/ec2-user/go/pkg/mod/github.com/kralicky/totem@v1.2.0/types.go:110 +0x376 + +goroutine 1676 [chan receive]: +runtime.gopark(0xc0024fc698?, 0x21c0d85?, 0x50?, 0xc7?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002fc1ef0 sp=0xc002fc1ed0 pc=0x4777ae +runtime.chanrecv(0xc002e0a960, 0xc002fc1fc0, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:583 +0x385 fp=0xc002fc1f68 sp=0xc002fc1ef0 pc=0x441325 +runtime.chanrecv1(0x0?, 0xc0024fc7d0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/chan.go:442 +0x12 fp=0xc002fc1f90 sp=0xc002fc1f68 pc=0x440f72 +github.com/rancher/opni/pkg/util/future.NewFromChannel[...].func1() + /home/ec2-user/opni/pkg/util/future/future.go:33 +0x48 fp=0xc002fc1fe0 sp=0xc002fc1f90 pc=0x20128e8 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002fc1fe8 sp=0xc002fc1fe0 pc=0x4ad421 +created by github.com/rancher/opni/pkg/util/future.NewFromChannel[...] in goroutine 1478 + /home/ec2-user/opni/pkg/util/future/future.go:32 +0x1ea + +goroutine 1656 [select]: +runtime.gopark(0xc001549f40?, 0x2?, 0xb4?, 0x1b?, 0xc001549ecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc001549d48 sp=0xc001549d28 pc=0x4777ae +runtime.selectgo(0xc001549f40, 0xc001549ec8, 0x0?, 0x0, 0x0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc001549e98 sp=0xc001549d48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002847680}, 0xc002949890, 0xc002fc7cb0, 0xc0022ee1e0, 0xc002fb9550) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc001549f70 sp=0xc001549e98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc001549fe0 sp=0xc001549f70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc001549fe8 sp=0xc001549fe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1505 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1686 [select]: +runtime.gopark(0xc0030f7e48?, 0x2?, 0x0?, 0x0?, 0xc0030f7dcc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0030f7c30 sp=0xc0030f7c10 pc=0x4777ae +runtime.selectgo(0xc0030f7e48, 0xc0030f7dc8, 0xc0030f7dd8?, 0x0, 0xc001f4ec88?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0030f7d80 sp=0xc0030f7c30 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0030b8e60, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc0030f7e78 sp=0xc0030f7d80 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0030d2150) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc0030f7f30 sp=0xc0030f7e78 pc=0x1102968 +google.golang.org/grpc/internal/transport.NewServerTransport.func2() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:341 +0x256 fp=0xc0030f7fe0 sp=0xc0030f7f30 pc=0x1128ed6 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0030f7fe8 sp=0xc0030f7fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1685 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:338 +0x2df9 + +goroutine 1627 [select]: +runtime.gopark(0xc00300ff40?, 0x2?, 0xb4?, 0x1b?, 0xc00300fecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc00300fd48 sp=0xc00300fd28 pc=0x4777ae +runtime.selectgo(0xc00300ff40, 0xc00300fec8, 0x4ad421?, 0x0, 0x1381849?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc00300fe98 sp=0xc00300fd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc0027ca5f0}, 0xc0004f1b90, 0xc002efb500, 0xc002047680, 0xc002ef2ed0) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc00300ff70 sp=0xc00300fe98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc00300ffe0 sp=0xc00300ff70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00300ffe8 sp=0xc00300ffe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1619 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1683 [select]: +runtime.gopark(0xc0011fff40?, 0x2?, 0xb4?, 0x1b?, 0xc0011ffecc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc0011ffd48 sp=0xc0011ffd28 pc=0x4777ae +runtime.selectgo(0xc0011fff40, 0xc0011ffec8, 0xc0030a9ed0?, 0x0, 0xc0030ab1c0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc0011ffe98 sp=0xc0011ffd48 pc=0x48916b +github.com/alitto/pond.worker({0x5b426e0, 0xc002cba0f0}, 0xc002949ad0, 0xc0030ab560, 0xc0022ef620, 0xc002f7f570) + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/worker.go:21 +0x176 fp=0xc0011fff70 sp=0xc0011ffe98 pc=0x12a82d6 +github.com/alitto/pond.(*WorkerPool).maybeStartWorker.func1() + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x7a fp=0xc0011fffe0 sp=0xc0011fff70 pc=0x12a735a +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0011fffe8 sp=0xc0011fffe0 pc=0x4ad421 +created by github.com/alitto/pond.(*WorkerPool).maybeStartWorker in goroutine 1536 + /home/ec2-user/go/pkg/mod/github.com/alitto/pond@v1.8.3/pond.go:423 +0x271 + +goroutine 1715 [select]: +runtime.gopark(0xc003135e70?, 0x2?, 0x0?, 0x0?, 0xc003135df4?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc003135c58 sp=0xc003135c38 pc=0x4777ae +runtime.selectgo(0xc003135e70, 0xc003135df0, 0xc002fe08c0?, 0x0, 0xc002f9a458?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc003135da8 sp=0xc003135c58 pc=0x48916b +google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002fea730, 0x1) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x1af fp=0xc003135ea0 sp=0xc003135da8 pc=0x1101ccf +google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002fe08c0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x128 fp=0xc003135f58 sp=0xc003135ea0 pc=0x1102968 +google.golang.org/grpc/internal/transport.newHTTP2Client.func6() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x168 fp=0xc003135fe0 sp=0xc003135f58 pc=0x1114408 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc003135fe8 sp=0xc003135fe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 1699 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x3974 + +goroutine 1633 [select]: +runtime.gopark(0xc002fadf70?, 0x2?, 0xb4?, 0x1b?, 0xc002fadebc?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002fadd20 sp=0xc002fadd00 pc=0x4777ae +runtime.selectgo(0xc002fadf70, 0xc002fadeb8, 0xc002000000?, 0x0, 0xa9fd34?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002fade70 sp=0xc002fadd20 pc=0x48916b +google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc002cc3b80, {0x5b426e0, 0xc002ef5f40}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x1bd fp=0xc002fadfa0 sp=0xc002fade70 pc=0x10e9d5d +google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x50 fp=0xc002fadfe0 sp=0xc002fadfa0 pc=0x10e9990 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002fadfe8 sp=0xc002fadfe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 609 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x24a + +goroutine 1687 [select]: +runtime.gopark(0xc002fbef70?, 0x4?, 0x28?, 0x8e?, 0xc002fbedd8?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002fbec00 sp=0xc002fbebe0 pc=0x4777ae +runtime.selectgo(0xc002fbef70, 0xc002fbedd0, 0x40a16a0?, 0x0, 0xc002f7fbe0?, 0x1) + /home/ec2-user/sdk/go1.21.0/src/runtime/select.go:327 +0x84b fp=0xc002fbed50 sp=0xc002fbec00 pc=0x48916b +google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002f2e4e0) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:1155 +0x3b2 fp=0xc002fbefc0 sp=0xc002fbed50 pc=0x11346d2 +google.golang.org/grpc/internal/transport.NewServerTransport.func4() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x34 fp=0xc002fbefe0 sp=0xc002fbefc0 pc=0x1128c54 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002fbefe8 sp=0xc002fbefe0 pc=0x4ad421 +created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 1685 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:344 +0x2e74 + +goroutine 1688 [IO wait]: +runtime.gopark(0x7ffff800000?, 0x2?, 0x2c?, 0xf0?, 0xffffffffffffffff?) + /home/ec2-user/sdk/go1.21.0/src/runtime/proc.go:398 +0xce fp=0xc002faf7d8 sp=0xc002faf7b8 pc=0x4777ae +runtime.netpollblock(0x4b0571?, 0x50e5a5?, 0x0?) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:564 +0xf7 fp=0xc002faf810 sp=0xc002faf7d8 pc=0x46ff37 +internal/poll.runtime_pollWait(0x7f42210c42d8, 0x72) + /home/ec2-user/sdk/go1.21.0/src/runtime/netpoll.go:343 +0x85 fp=0xc002faf830 sp=0xc002faf810 pc=0x4a7205 +internal/poll.(*pollDesc).wait(0xc002da12a0, 0xc0030d6000?, 0x0) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:84 +0xb1 fp=0xc002faf878 sp=0xc002faf830 pc=0x50cbf1 +internal/poll.(*pollDesc).waitRead(...) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc002da1280, {0xc0030d6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/internal/poll/fd_unix.go:164 +0x3e5 fp=0xc002faf968 sp=0xc002faf878 pc=0x50e5a5 +net.(*netFD).Read(0xc002da1280, {0xc0030d6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/fd_posix.go:55 +0x4b fp=0xc002faf9c8 sp=0xc002faf968 pc=0x89342b +net.(*conn).Read(0xc001f4ec88, {0xc0030d6000, 0x8000, 0x8000}) + /home/ec2-user/sdk/go1.21.0/src/net/net.go:179 +0xad fp=0xc002fafa50 sp=0xc002faf9c8 pc=0x8b176d +net.(*TCPConn).Read(0xc001f4ec88, {0xc0030d6000, 0x8000, 0x8000}) + :1 +0x54 fp=0xc002fafa98 sp=0xc002fafa50 pc=0x8d72f4 +bufio.(*Reader).Read(0xc00309bc20, {0xc002f4f620, 0x9, 0x9}) + /home/ec2-user/sdk/go1.21.0/src/bufio/bufio.go:244 +0x4be fp=0xc002fafb58 sp=0xc002fafa98 pc=0x5f943e +io.ReadAtLeast({0x5b0f0a0, 0xc00309bc20}, {0xc002f4f620, 0x9, 0x9}, 0x9) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:335 +0xd0 fp=0xc002fafba0 sp=0xc002fafb58 pc=0x500a90 +io.ReadFull(...) + /home/ec2-user/sdk/go1.21.0/src/io/io.go:354 +golang.org/x/net/http2.readFrameHeader({0xc002f4f620, 0x9, 0x9}, {0x5b0f0a0, 0xc00309bc20}) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:237 +0x9b fp=0xc002fafc40 sp=0xc002fafba0 pc=0xe670fb +golang.org/x/net/http2.(*Framer).ReadFrame(0xc002f4f5e0) + /home/ec2-user/go/pkg/mod/golang.org/x/net@v0.14.0/http2/frame.go:498 +0xf5 fp=0xc002fafd78 sp=0xc002fafc40 pc=0xe681d5 +google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002f2e4e0, 0x4ad421?, 0xc000f3f7d0?) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_server.go:642 +0x13b fp=0xc002fafec8 sp=0xc002fafd78 pc=0x112e2bb +google.golang.org/grpc.(*Server).serveStreams(0xc00191c1e0, {0x5b54e00?, 0xc002f2e4e0}) + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:969 +0x23c fp=0xc002faff80 sp=0xc002fafec8 pc=0x117905c +google.golang.org/grpc.(*Server).handleRawConn.func1() + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:912 +0x59 fp=0xc002faffe0 sp=0xc002faff80 pc=0x1178539 +runtime.goexit() + /home/ec2-user/sdk/go1.21.0/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc002faffe8 sp=0xc002faffe0 pc=0x4ad421 +created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 1685 + /home/ec2-user/go/pkg/mod/google.golang.org/grpc@v1.57.0/server.go:911 +0x245 + +Ginkgo ran 1 suite in 9.808135734s + +Test Suite Failed diff --git a/test/plugins/alerting/routing_test.go b/test/plugins/alerting/routing_test.go index 3768c79842..ad083ea688 100644 --- a/test/plugins/alerting/routing_test.go +++ b/test/plugins/alerting/routing_test.go @@ -18,9 +18,9 @@ import ( "github.com/rancher/opni/pkg/alerting/drivers/routing" "github.com/rancher/opni/pkg/alerting/shared" alertingv1 "github.com/rancher/opni/pkg/apis/alerting/v1" + "github.com/rancher/opni/pkg/test" "github.com/rancher/opni/pkg/test/alerting" "github.com/rancher/opni/pkg/test/freeport" - "github.com/rancher/opni/pkg/test/testruntime" "github.com/rancher/opni/pkg/util" "github.com/samber/lo" "golang.org/x/exp/slices" @@ -29,225 +29,226 @@ import ( var defaultHook *alerting.MockIntegrationWebhookServer -func init() { - testruntime.IfIntegration(func() { - BuildRoutingLogicTest( - func() routing.OpniRouting { - defaultHooks := alerting.NewWebhookMemoryServer(env, "webhook") - defaultHook = defaultHooks - cfg := config.WebhookConfig{ - NotifierConfig: config.NotifierConfig{ - VSendResolved: false, - }, - URL: &amCfg.URL{ - URL: util.Must(url.Parse(defaultHook.GetWebhook())), - }, - } - return routing.NewOpniRouterV1(cfg) - }, - ) +var _ = Describe("Alerting routing logic translation to physical dispatching", Ordered, Label("integration"), func() { + var alertingClient client.AlertingClient + var alertingClient2 client.AlertingClient + var alertingClient3 client.AlertingClient + var router routing.OpniRouting + var env *test.Environment + var tmpConfigDir string + + BeforeAll(func() { + env = &test.Environment{} + Expect(env).NotTo(BeNil()) + Expect(env.Start()).To(Succeed()) + DeferCleanup(env.Stop) + tmpConfigDir = env.GenerateNewTempDirectory("alertmanager-config") + Expect(tmpConfigDir).NotTo(Equal("")) + + routerConstructor := func() routing.OpniRouting { + defaultHooks := alerting.NewWebhookMemoryServer(env, "webhook") + defaultHook = defaultHooks + cfg := config.WebhookConfig{ + NotifierConfig: config.NotifierConfig{ + VSendResolved: false, + }, + URL: &amCfg.URL{ + URL: util.Must(url.Parse(defaultHook.GetWebhook())), + }, + } + return routing.NewOpniRouterV1(cfg) + } + + router = routerConstructor() + Expect(router).NotTo(BeNil()) }) -} -func BuildRoutingLogicTest( - routerConstructor func() routing.OpniRouting, -) bool { - return Describe("Alerting routing logic translation to physical dispatching", Ordered, Label("integration"), func() { - var alertingClient client.AlertingClient - var alertingClient2 client.AlertingClient - var alertingClient3 client.AlertingClient - When("setting namespace specs on the routing tree", func() { - step := "initial" - var router routing.OpniRouting - BeforeAll(func() { - Expect(env).NotTo(BeNil()) - router = routerConstructor() - Expect(router).NotTo(BeNil()) - }) - AfterEach(func() { - By(fmt.Sprintf("%s step: expecting that the router can build the config", step)) - currentCfg, err := router.BuildConfig() - Expect(err).To(Succeed()) - By(fmt.Sprintf("%s step: expecting that the formed alertmanager config is correct", step)) - fp := freeport.GetFreePort() + When("setting namespace specs on the routing tree", func() { + step := "initial" - alerting.ExpectAlertManagerConfigToBeValid(env.Context(), env, tmpConfigDir, step+".yaml", currentCfg, fp) - }) + AfterEach(func() { + By(fmt.Sprintf("%s step: expecting that the router can build the config", step)) + currentCfg, err := router.BuildConfig() + Expect(err).To(Succeed()) + By(fmt.Sprintf("%s step: expecting that the formed alertmanager config is correct", step)) + fp := freeport.GetFreePort() - It("should be able to dynamically update alert routing", func() { - step = "dynamic-alert-routing" - tmpConfigDir := env.GenerateNewTempDirectory("webhook") - err := os.MkdirAll(tmpConfigDir, 0755) - Expect(err).To(Succeed()) - By("Creating some test webhook servers") + alerting.ExpectAlertManagerConfigToBeValid(env.Context(), env, tmpConfigDir, step+".yaml", currentCfg, fp) + }) - servers := alerting.CreateWebhookServer(env, 3) - server1, server2, server3 := servers[0], servers[1], servers[2] + It("should be able to dynamically update alert routing", func() { + step = "dynamic-alert-routing" + tmpConfigDir := env.GenerateNewTempDirectory("webhook") + err := os.MkdirAll(tmpConfigDir, 0755) + Expect(err).To(Succeed()) + By("Creating some test webhook servers") - condId1, condId2, condId3 := uuid.New().String(), uuid.New().String(), uuid.New().String() - ns := "test" - By("routing to a subset of the test webhook servers") - details1 := &alertingv1.EndpointImplementation{ - Title: "test1", - Body: "test1", - } - details2 := &alertingv1.EndpointImplementation{ - Title: "test2", - Body: "test2", - } - details3 := &alertingv1.EndpointImplementation{ - Title: "test3", - Body: "test3", - } - suiteSpec := &testSpecSuite{ - name: "dynamic-alert-routing", - defaultServer: defaultHook, - specs: []*testSpec{ - { - namespace: ns, - id: condId1, - servers: []*alerting.MockIntegrationWebhookServer{server1}, - details: details1, - }, - { - namespace: ns, - id: condId2, - servers: []*alerting.MockIntegrationWebhookServer{server1, server2}, - details: details2, - }, - { - namespace: ns, - id: condId3, - servers: []*alerting.MockIntegrationWebhookServer{server1, server2, server3}, - details: details3, - }, - }, - } - By("setting the router to the namespace specs") - for _, spec := range suiteSpec.specs { - endpoints := lo.Map( - spec.servers, - func(server *alerting.MockIntegrationWebhookServer, _ int) *alertingv1.FullAttachedEndpoint { - return &alertingv1.FullAttachedEndpoint{ - AlertEndpoint: server.Endpoint(), - EndpointId: server.Endpoint().Id, - Details: spec.details, - } - }) - err = router.SetNamespaceSpec( - spec.namespace, - spec.id, - &alertingv1.FullAttachedEndpoints{ - Items: endpoints, - Details: spec.details, - InitialDelay: durationpb.New(time.Second * 1), - ThrottlingDuration: durationpb.New(time.Second * 1), - }, - ) - Expect(err).To(Succeed()) - } + servers := alerting.CreateWebhookServer(env, 3) + server1, server2, server3 := servers[0], servers[1], servers[2] - By("running alertmanager with this config") - amPort, ca := alerting.RunAlertManager(env, router, tmpConfigDir, step+".yaml") - alertingClient = client.NewClient( - nil, - fmt.Sprintf("http://localhost:%d", amPort), - fmt.Sprintf("http://localhost:%d", 0), - ) - defer ca() - By("sending alerts to each condition in the router") - for _, spec := range suiteSpec.specs { - err := alertingClient.AlertClient().PostAlarm(context.TODO(), client.AlertObject{ - Id: spec.id, - Labels: map[string]string{ - ns: spec.id, - }, - Annotations: map[string]string{}, + condId1, condId2, condId3 := uuid.New().String(), uuid.New().String(), uuid.New().String() + ns := "test" + By("routing to a subset of the test webhook servers") + details1 := &alertingv1.EndpointImplementation{ + Title: "test1", + Body: "test1", + } + details2 := &alertingv1.EndpointImplementation{ + Title: "test2", + Body: "test2", + } + details3 := &alertingv1.EndpointImplementation{ + Title: "test3", + Body: "test3", + } + suiteSpec := &testSpecSuite{ + name: "dynamic-alert-routing", + defaultServer: defaultHook, + specs: []*testSpec{ + { + namespace: ns, + id: condId1, + servers: []*alerting.MockIntegrationWebhookServer{server1}, + details: details1, + }, + { + namespace: ns, + id: condId2, + servers: []*alerting.MockIntegrationWebhookServer{server1, server2}, + details: details2, + }, + { + namespace: ns, + id: condId3, + servers: []*alerting.MockIntegrationWebhookServer{server1, server2, server3}, + details: details3, + }, + }, + } + By("setting the router to the namespace specs") + for _, spec := range suiteSpec.specs { + endpoints := lo.Map( + spec.servers, + func(server *alerting.MockIntegrationWebhookServer, _ int) *alertingv1.FullAttachedEndpoint { + return &alertingv1.FullAttachedEndpoint{ + AlertEndpoint: server.Endpoint(), + EndpointId: server.Endpoint().Id, + Details: spec.details, + } }) - Expect(err).To(Succeed()) - } - Eventually(func() error { - return suiteSpec.ExpectAlertsToBeRouted(amPort) - }, time.Second*30, time.Second*1).Should(Succeed()) - ca() - server1.ClearBuffer() - server2.ClearBuffer() - server3.ClearBuffer() - defaultHook.ClearBuffer() + err = router.SetNamespaceSpec( + spec.namespace, + spec.id, + &alertingv1.FullAttachedEndpoints{ + Items: endpoints, + Details: spec.details, + InitialDelay: durationpb.New(time.Second * 1), + ThrottlingDuration: durationpb.New(time.Second * 1), + }, + ) + Expect(err).To(Succeed()) + } - By("deleting a random server endpoint") - // ok - err = router.DeleteEndpoint(suiteSpec.specs[0].servers[0].Endpoint().Id) + By("running alertmanager with this config") + amPort, ca := alerting.RunAlertManager(env, router, tmpConfigDir, step+".yaml") + alertingClient = client.NewClient( + nil, + fmt.Sprintf("http://localhost:%d", amPort), + fmt.Sprintf("http://localhost:%d", 0), + ) + defer ca() + By("sending alerts to each condition in the router") + for _, spec := range suiteSpec.specs { + err := alertingClient.AlertClient().PostAlarm(context.TODO(), client.AlertObject{ + Id: spec.id, + Labels: map[string]string{ + ns: spec.id, + }, + Annotations: map[string]string{}, + }) Expect(err).To(Succeed()) - for _, spec := range suiteSpec.specs { - spec.servers = spec.servers[1:] - } + } + Eventually(func() error { + return suiteSpec.ExpectAlertsToBeRouted(amPort) + }, time.Second*30, time.Second*1).Should(Succeed()) + ca() + server1.ClearBuffer() + server2.ClearBuffer() + server3.ClearBuffer() + defaultHook.ClearBuffer() - amPort2, ca2 := alerting.RunAlertManager(env, router, tmpConfigDir, step+".yaml") - alertingClient2 = client.NewClient( - nil, - fmt.Sprintf("http://localhost:%d", amPort2), - fmt.Sprintf("http://localhost:%d", 0), - ) - defer ca2() - By("sending alerts to each condition in the router") - for _, spec := range suiteSpec.specs { - err := alertingClient2.AlertClient().PostAlarm(context.TODO(), client.AlertObject{ - Id: spec.id, - Labels: map[string]string{ - ns: spec.id, - }, - Annotations: map[string]string{}, - }) - Expect(err).To(Succeed()) - } - Eventually(func() error { - return suiteSpec.ExpectAlertsToBeRouted(amPort2) - }, time.Second*30, time.Second*1).Should(Succeed()) - ca2() + By("deleting a random server endpoint") + // ok + err = router.DeleteEndpoint(suiteSpec.specs[0].servers[0].Endpoint().Id) + Expect(err).To(Succeed()) + for _, spec := range suiteSpec.specs { + spec.servers = spec.servers[1:] + } - By("updating an endpoint to another endpoint") + amPort2, ca2 := alerting.RunAlertManager(env, router, tmpConfigDir, step+".yaml") + alertingClient2 = client.NewClient( + nil, + fmt.Sprintf("http://localhost:%d", amPort2), + fmt.Sprintf("http://localhost:%d", 0), + ) + defer ca2() + By("sending alerts to each condition in the router") + for _, spec := range suiteSpec.specs { + err := alertingClient2.AlertClient().PostAlarm(context.TODO(), client.AlertObject{ + Id: spec.id, + Labels: map[string]string{ + ns: spec.id, + }, + Annotations: map[string]string{}, + }) + Expect(err).To(Succeed()) + } + Eventually(func() error { + return suiteSpec.ExpectAlertsToBeRouted(amPort2) + }, time.Second*30, time.Second*1).Should(Succeed()) + ca2() - server1.ClearBuffer() - server2.ClearBuffer() - server3.ClearBuffer() - defaultHook.ClearBuffer() + By("updating an endpoint to another endpoint") - err = router.UpdateEndpoint(server2.Endpoint().Id, server1.Endpoint()) - Expect(err).To(Succeed()) - for _, spec := range suiteSpec.specs { - if len(spec.servers) != 0 { - spec.servers[0] = server1 - } - } + server1.ClearBuffer() + server2.ClearBuffer() + server3.ClearBuffer() + defaultHook.ClearBuffer() - By("send an an alert to each specs") - amPort3, ca3 := alerting.RunAlertManager(env, router, tmpConfigDir, step+".yaml") - defer ca3() - alertingClient3 = client.NewClient( - nil, - fmt.Sprintf("http://localhost:%d", amPort3), - fmt.Sprintf("http://localhost:%d", 0), - ) - By("sending alerts to each condition in the router") - for _, spec := range suiteSpec.specs { - err := alertingClient3.AlertClient().PostAlarm(context.TODO(), client.AlertObject{ - Id: spec.id, - Labels: map[string]string{ - ns: spec.id, - }, - Annotations: map[string]string{}, - }) - Expect(err).To(Succeed()) + err = router.UpdateEndpoint(server2.Endpoint().Id, server1.Endpoint()) + Expect(err).To(Succeed()) + for _, spec := range suiteSpec.specs { + if len(spec.servers) != 0 { + spec.servers[0] = server1 } - Eventually(func() error { - return suiteSpec.ExpectAlertsToBeRouted(amPort3) - }, time.Second*30, time.Second*1).Should(Succeed()) - ca3() - }) + } + + By("send an an alert to each specs") + amPort3, ca3 := alerting.RunAlertManager(env, router, tmpConfigDir, step+".yaml") + defer ca3() + alertingClient3 = client.NewClient( + nil, + fmt.Sprintf("http://localhost:%d", amPort3), + fmt.Sprintf("http://localhost:%d", 0), + ) + By("sending alerts to each condition in the router") + for _, spec := range suiteSpec.specs { + err := alertingClient3.AlertClient().PostAlarm(context.TODO(), client.AlertObject{ + Id: spec.id, + Labels: map[string]string{ + ns: spec.id, + }, + Annotations: map[string]string{}, + }) + Expect(err).To(Succeed()) + } + Eventually(func() error { + return suiteSpec.ExpectAlertsToBeRouted(amPort3) + }, time.Second*30, time.Second*1).Should(Succeed()) + ca3() }) }) -} +}) type testSpecSuite struct { name string diff --git a/test/plugins/alerting/slo_test.go b/test/plugins/alerting/slo_test.go new file mode 100644 index 0000000000..6c61f78ec0 --- /dev/null +++ b/test/plugins/alerting/slo_test.go @@ -0,0 +1,565 @@ +package alerting_test + +import ( + "context" + "fmt" + "time" + + "github.com/golang/mock/gomock" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "github.com/rancher/opni/pkg/apis/core/v1" + managementv1 "github.com/rancher/opni/pkg/apis/management/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + "github.com/rancher/opni/pkg/capabilities" + "github.com/rancher/opni/pkg/capabilities/wellknown" + "github.com/rancher/opni/pkg/slo/backend" + "github.com/rancher/opni/pkg/test" + mockslo "github.com/rancher/opni/pkg/test/mock/slo" + "github.com/rancher/opni/pkg/test/testutil" + "github.com/rancher/opni/pkg/util" + "github.com/rancher/opni/plugins/alerting/apis/alertops" + "github.com/rancher/opni/plugins/alerting/pkg/alerting/slo/v1" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/emptypb" +) + +const ( + mockSource = "slo" + errorSource = "error" +) + +var _ = Describe("SLO Alerting", Ordered, Label("integration"), func() { + var sloClient slov1.SLOClient + var env *test.Environment + BeforeAll(func() { + env = &test.Environment{} + Expect(env).NotTo(BeNil()) + Expect(env.Start()).To(Succeed()) + DeferCleanup(env.Stop) + + ctrl := gomock.NewController(GinkgoT()) + + errStore := mockslo.NewMockSLOStore(ctrl) + + errDiscovery := mockslo.NewMockServiceBackend(ctrl) + + errStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return( + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + errStore.EXPECT().Delete(gomock.Any(), gomock.Any()).Return( + fmt.Errorf("some error"), + ).AnyTimes() + + errStore.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return( + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + errStore.EXPECT().Clone(gomock.Any(), gomock.Any()).Return( + nil, + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + errStore.EXPECT().MultiClusterClone(gomock.Any(), gomock.Any(), gomock.Any()).Return( + nil, + nil, + []error{fmt.Errorf("some error")}, + ).AnyTimes() + + errStore.EXPECT().Status(gomock.Any(), gomock.Any()).Return( + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + errStore.EXPECT().Preview(gomock.Any(), gomock.Any()).Return( + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + errDiscovery.EXPECT().ListEvents(gomock.Any(), gomock.Any()).Return( + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + errDiscovery.EXPECT().ListMetrics(gomock.Any(), gomock.Any()).Return( + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + errDiscovery.EXPECT().ListServices(gomock.Any(), gomock.Any()).Return( + nil, + fmt.Errorf("some error"), + ).AnyTimes() + + sloStore := mockslo.NewMockSLOStore(ctrl) + sloDiscovery := mockslo.NewMockServiceBackend(ctrl) + + sloDiscovery.EXPECT().ListServices(gomock.Any(), gomock.Any()).Return( + &slov1.ServiceList{ + Items: []*slov1.Service{ + { + ServiceId: "test-service", + ClusterId: "some-cluster", + }, + }, + }, + nil, + ).AnyTimes() + + sloDiscovery.EXPECT().ListMetrics(gomock.Any(), gomock.Any()).Return( + &slov1.MetricGroupList{ + GroupNameToMetrics: map[string]*slov1.MetricList{ + "testgroup": { + Items: []*slov1.Metric{ + { + Id: "testmetric", + }, + }, + }, + }, + }, + nil, + ).AnyTimes() + + sloDiscovery.EXPECT().ListEvents(gomock.Any(), gomock.Any()).Return( + &slov1.EventList{ + Items: []*slov1.Event{ + { + Key: "test-event", + Vals: []string{"a", "b", "c"}, + }, + }, + }, + nil, + ).AnyTimes() + + sloStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return( + &corev1.Reference{ + Id: "test-slo", + }, + nil, + ).AnyTimes() + + sloStore.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return( + &slov1.SLOData{ + Id: "test-slo", + SLO: util.ProtoClone(updatedMockSLO), + }, + nil, + ).AnyTimes() + + sloStore.EXPECT().Delete(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + sloStore.EXPECT().Clone(gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, toClone *slov1.SLOData) (*corev1.Reference, *slov1.SLOData, error) { + id := uuid.New().String() + clonedData := util.ProtoClone(toClone) + clonedData.Id = id + return &corev1.Reference{ + Id: id, + }, + clonedData, + nil + }).AnyTimes() + + sloStore.EXPECT().MultiClusterClone(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func( + _ context.Context, + slo *slov1.SLOData, + toClusters []*corev1.Reference, + ) ([]*corev1.Reference, []*slov1.SLOData, []error) { + ids := []*corev1.Reference{} + datas := []*slov1.SLOData{} + errors := []error{} + for _, cluster := range toClusters { + uuid := uuid.New().String() + newSLO := util.ProtoClone( + slo.GetSLO(), + ) + newSLO.ClusterId = cluster.Id + newSLO.Name += "-clone" + ids = append(ids, &corev1.Reference{Id: uuid}) + datas = append(datas, &slov1.SLOData{ + Id: uuid, + SLO: newSLO, + }) + errors = append(errors, nil) + } + return ids, datas, errors + }).AnyTimes() + + sloStore.EXPECT().Status(gomock.Any(), gomock.Any()).Return( + &slov1.SLOStatus{ + State: slov1.SLOStatusState_Creating, + }, + nil, + ).AnyTimes() + + sloStore.EXPECT().Preview(gomock.Any(), gomock.Any()).Return( + &slov1.SLOPreviewResponse{ + PlotVector: &slov1.PlotVector{ + Objective: 99.9, + Items: []*slov1.DataPoint{}, + Windows: []*slov1.AlertFiringWindows{}, + }, + }, + nil, + ).AnyTimes() + + alertClusterClient := alertops.NewAlertingAdminClient(env.ManagementClientConn()) + mgmtClient := env.NewManagementClient() + + mockDatasource := backend.NewSLODatasource(sloStore, sloDiscovery, func(_ context.Context, _ *corev1.Reference) error { + return nil + }) + + errDatasource := backend.NewSLODatasource(errStore, errDiscovery, func(_ context.Context, _ *corev1.Reference) error { + return nil + }) + + slo.RegisterDatasource(mockSource, mockDatasource) + slo.RegisterDatasource(errorSource, errDatasource) + + _, err := alertClusterClient.InstallCluster(env.Context(), &emptypb.Empty{}) + Expect(err).To(BeNil()) + + certsInfo, err := mgmtClient.CertsInfo(context.Background(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + fingerprint := certsInfo.Chain[len(certsInfo.Chain)-1].Fingerprint + Expect(fingerprint).NotTo(BeEmpty()) + token, err := mgmtClient.CreateBootstrapToken(context.Background(), &managementv1.CreateBootstrapTokenRequest{ + Ttl: durationpb.New(1 * time.Hour), + }) + Expect(err).NotTo(HaveOccurred()) + _, errC := env.StartAgent("agent", token, []string{fingerprint}) + Eventually(errC, time.Second*10, time.Millisecond*200).Should(Receive(BeNil())) + + _, errC2 := env.StartAgent("agent2", token, []string{fingerprint}) + Eventually(errC2, time.Second*10, time.Millisecond*200).Should(Receive(BeNil())) + + Eventually(func() error { + status, err := alertClusterClient.GetClusterStatus(env.Context(), &emptypb.Empty{}) + if err != nil { + return err + } + if status.State != alertops.InstallState_Installed { + return fmt.Errorf("alerting cluster install state is %s", status.State.String()) + } + return nil + }, time.Second*5, time.Millisecond*200).Should(Succeed()) + + sloClient = slov1.NewSLOClient(env.ManagementClientConn()) + }) + + When("interacting with the SLO server", func() { + It("should list the available datasources", func() { + datasources, err := sloClient.ListDatasources(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(datasources.Items).To(ConsistOf([]string{errorSource, mockSource, "monitoring"})) + }) + }) + + When("Managing SLOs with the mocked datasource", func() { + It("should return a list of available services", func() { + svcList, err := sloClient.ListServices(env.Context(), &slov1.ListServicesRequest{ + Datasource: mockSource, + ClusterId: "some-cluster", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(svcList.Items).To(HaveLen(1)) + Expect(svcList.Items[0]).To( + Equal(&slov1.Service{ + ServiceId: "test-service", + ClusterId: "some-cluster", + }), + ) + }) + + It("should return a list of available metrics", func() { + metricList, err := sloClient.ListMetrics(env.Context(), &slov1.ListMetricsRequest{ + Datasource: mockSource, + ClusterId: "some-cluster", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(metricList.GroupNameToMetrics).To(HaveLen(1)) + Expect(metricList.GroupNameToMetrics["testgroup"].Items).To( + Equal( + []*slov1.Metric{ + { + Id: "testmetric", + }, + }, + ), + ) + }) + + It("should return a list of available events", func() { + eventList, err := sloClient.ListEvents(env.Context(), &slov1.ListEventsRequest{ + Datasource: mockSource, + ClusterId: "some-cluster", + ServiceId: "test-service", + MetricId: "testmetric", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(eventList.Items).To(HaveLen(1)) + Expect(eventList.Items[0]).To( + Equal(&slov1.Event{ + Key: "test-event", + Vals: []string{"a", "b", "c"}, + }), + ) + }) + + It("should create an SLO", func() { + sloImpl := util.ProtoClone(mockSLO) + sloRef, err := sloClient.CreateSLO(env.Context(), &slov1.CreateSLORequest{ + Slo: sloImpl, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(sloRef).NotTo(BeNil()) + + By("verifying we can fetch the SLO") + data, err := sloClient.GetSLO(env.Context(), sloRef) + Expect(err).NotTo(HaveOccurred()) + + Expect(data.SLO).To(testutil.ProtoEqual((sloImpl))) + + By("verifying the SLO list only contains one element") + sloList, err := sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(sloList.Items).To(HaveLen(1)) + }) + + Specify("when the SLOStore fails to create an SLO, the management server should not have it", func() { + sloList, err := sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + originalLen := len(sloList.Items) + + sloImpl := util.ProtoClone(mockSLO) + sloImpl.Datasource = errorSource + createSLOFailure := &slov1.CreateSLORequest{ + Slo: sloImpl, + } + + sloRef, err := sloClient.CreateSLO(env.Context(), createSLOFailure) + Expect(err).To(HaveOccurred()) + Expect(sloRef).To(BeNil()) + + sloList, err = sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(sloList.Items).To(HaveLen(originalLen)) + }) + + It("should update the SLO details", func() { + sloImpl := util.ProtoClone(mockSLO) + _, err := sloClient.UpdateSLO(env.Context(), &slov1.SLOData{ + Id: "test-slo", + SLO: sloImpl, + }) + Expect(err).NotTo(HaveOccurred()) + + data, err := sloClient.GetSLO(env.Context(), &corev1.Reference{ + Id: "test-slo", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(data.SLO).To(testutil.ProtoEqual(updatedMockSLO)) + }) + + It("should clone an SLO", func() { + sloList, err := sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(sloList.Items).NotTo(HaveLen(0)) + + toClone := sloList.GetItems()[0] + + clone, err := sloClient.CloneSLO(env.Context(), &corev1.Reference{Id: toClone.Id}) + Expect(err).NotTo(HaveOccurred()) + Expect(clone.SLO).To(testutil.ProtoEqual(toClone.GetSLO())) + }) + + It("should clone an SLO across clusters", func() { + sloList, err := sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(sloList.Items).NotTo(HaveLen(0)) + + originalLen := len(sloList.Items) + + toClone := sloList.GetItems()[0] + + failures, err := sloClient.CloneToClusters(env.Context(), &slov1.MultiClusterSLO{ + CloneId: &corev1.Reference{Id: toClone.GetId()}, + Clusters: []*corev1.Reference{ + {Id: "agent"}, + {Id: "agent2"}, + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(failures.GetFailures()).To(HaveLen(0)) + + sloList, err = sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(sloList.Items).To(HaveLen(originalLen + 2)) + + }) + + It("should error when the fetching the status of non-existent SLO", func() { + status, err := sloClient.GetSLOStatus(env.Context(), &corev1.Reference{ + Id: "definitely does not exists", + }) + + Expect(err).To(HaveOccurred()) + Expect(status).To(BeNil()) + }) + + It("should report the status of running SLOs", func() { + slos, err := sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(slos.GetItems()).NotTo(HaveLen(0)) + + for _, slo := range slos.GetItems() { + status, err := sloClient.GetSLOStatus(env.Context(), &corev1.Reference{ + Id: slo.GetId(), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(status).NotTo(BeNil()) + Expect(status.State).To(Equal(slov1.SLOStatusState_Creating)) + } + }) + + It("should return a preview of an SLO definition", func() { + payload := util.ProtoClone(mockSLO) + vector, err := sloClient.Preview(env.Context(), &slov1.CreateSLORequest{ + Slo: payload, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(vector.GetPlotVector()).NotTo(BeNil()) + }) + + It("should delete SLOs", func() { + sloList, err := sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(len(sloList.Items)).To(BeNumerically(">", 0)) + + for _, item := range sloList.GetItems() { + _, err = sloClient.DeleteSLO(env.Context(), &corev1.Reference{ + Id: item.GetId(), + }) + Expect(err).NotTo(HaveOccurred()) + } + + sloList, err = sloClient.ListSLOs(env.Context(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + Expect(sloList.Items).To(HaveLen(0)) + }) + }) + + When("Managing SLOs with the metrics datasource", func() { + It("Should return failed precondition when metrics is not installed", func() { + mgmtClient := env.NewManagementClient() + cluster, err := mgmtClient.GetCluster(env.Context(), &corev1.Reference{Id: "agent"}) + Expect(err).NotTo(HaveOccurred()) + Expect(capabilities.Has(cluster, capabilities.Cluster(wellknown.CapabilityMetrics))).To(BeFalse()) + + _, err = sloClient.CreateSLO(env.Context(), &slov1.CreateSLORequest{ + Slo: metricsSLO, + }) + Expect(err).To(HaveOccurred()) + Expect(err).To(testutil.MatchStatusCode(codes.FailedPrecondition)) + + _, err = sloClient.CreateSLO(env.Context(), &slov1.CreateSLORequest{ + Slo: util.ProtoClone(metricsSLO), + }) + Expect(err).To(HaveOccurred()) + Expect(err).To(testutil.MatchStatusCode(codes.FailedPrecondition)) + + _, err = sloClient.ListServices(env.Context(), &slov1.ListServicesRequest{ + Datasource: "monitoring", + ClusterId: "agent", + }) + Expect(err).To(HaveOccurred()) + Expect(err).To(testutil.MatchStatusCode(codes.FailedPrecondition)) + + _, err = sloClient.ListEvents(env.Context(), &slov1.ListEventsRequest{ + Datasource: "monitoring", + ClusterId: "agent", + ServiceId: "test-service", + MetricId: "testmetric", + }) + Expect(err).To(HaveOccurred()) + + _, err = sloClient.ListMetrics(env.Context(), &slov1.ListMetricsRequest{ + Datasource: "monitoring", + ClusterId: "agent", + ServiceId: "test-service", + }) + Expect(err).To(HaveOccurred()) + Expect(err).To(testutil.MatchStatusCode(codes.FailedPrecondition)) + }) + }) +}) + +var ( + mockSLO = &slov1.ServiceLevelObjective{ + Name: "test-slo-success", + ClusterId: "agent", + ServiceId: "test-service", + GoodMetricName: "testmetric", + GoodEvents: []*slov1.Event{ + { + Key: "a", + Vals: []string{"a", "b", "c"}, + }, + }, + TotalMetricName: "testmetric", + TotalEvents: []*slov1.Event{}, + Target: &slov1.Target{Value: 99.9}, + SloPeriod: "30d", + BudgetingInterval: durationpb.New(time.Minute * 5), + Datasource: mockSource, + } + + metricsSLO = &slov1.ServiceLevelObjective{ + Name: "test-slo-success", + ClusterId: "agent", + ServiceId: "test-service", + GoodMetricName: "testmetric", + GoodEvents: []*slov1.Event{ + { + Key: "a", + Vals: []string{"a", "b", "c"}, + }, + }, + TotalMetricName: "testmetric", + TotalEvents: []*slov1.Event{}, + Target: &slov1.Target{Value: 99.9}, + SloPeriod: "30d", + BudgetingInterval: durationpb.New(time.Minute * 5), + Datasource: "monitoring", + } + + updatedMockSLO = &slov1.ServiceLevelObjective{ + Name: "test-slo-success2", + ClusterId: "agent", + ServiceId: "test-service2", + GoodMetricName: "testmetric2", + GoodEvents: []*slov1.Event{ + { + Key: "a2", + Vals: []string{"a2", "b2", "c2"}, + }, + }, + TotalMetricName: "testmetric2", + TotalEvents: []*slov1.Event{}, + Target: &slov1.Target{Value: 99.9}, + SloPeriod: "30d", + BudgetingInterval: durationpb.New(time.Minute * 5), + Datasource: mockSource, + } +) diff --git a/test/plugins/alerting/testlog.txt b/test/plugins/alerting/testlog.txt new file mode 100644 index 0000000000..ef7f96732c --- /dev/null +++ b/test/plugins/alerting/testlog.txt @@ -0,0 +1,5437 @@ +Running Suite: Alerting Suite - /home/ec2-user/opni/test/plugins/alerting +========================================================================= +Random Seed: 1692063585 + +Will run 40 of 41 specs +------------------------------ +[BeforeSuite]  +/home/ec2-user/opni/test/plugins/alerting/alerting_suite_test.go:46 + 2023-08-15T01:39:55Z INFO test.env Starting test environment + 2023-08-15T01:39:55Z INFO test.env Copying 7 files from embedded testdata/cortex to /tmp/opni-test-2132242568/cortex + 2023-08-15T01:39:55Z INFO test.env Waiting for jetstream to start... + [231479] 2023/08/15 01:39:55.563722 [INF] Starting nats-server + [231479] 2023/08/15 01:39:55.563769 [INF] Version: 2.9.21 + [231479] 2023/08/15 01:39:55.563772 [INF] Git: [b2e7725] + [231479] 2023/08/15 01:39:55.563798 [INF] Name: NDB2A7RMGWYYYSG3TTANYY53LKD2LU7AGFTQ7AIYHAYGKNAHXFE3EGX7 + [231479] 2023/08/15 01:39:55.563806 [INF] Node: B5UBIPgv + [231479] 2023/08/15 01:39:55.563808 [INF] ID: NDB2A7RMGWYYYSG3TTANYY53LKD2LU7AGFTQ7AIYHAYGKNAHXFE3EGX7 + [231479] 2023/08/15 01:39:55.563817 [INF] Using configuration file: /tmp/opni-test-2132242568/jetstream/jetstream.conf + [231479] 2023/08/15 01:39:55.564279 [INF] Starting JetStream + [231479] 2023/08/15 01:39:55.564540 [INF] _ ___ _____ ___ _____ ___ ___ _ __ __ + [231479] 2023/08/15 01:39:55.564554 [INF] _ | | __|_ _/ __|_ _| _ \ __| /_\ | \/ | + [231479] 2023/08/15 01:39:55.564557 [INF] | || | _| | | \__ \ | | | / _| / _ \| |\/| | + [231479] 2023/08/15 01:39:55.564560 [INF] \__/|___| |_| |___/ |_| |_|_\___/_/ \_\_| |_| + [231479] 2023/08/15 01:39:55.564563 [INF] + [231479] 2023/08/15 01:39:55.564566 [INF] https://docs.nats.io/jetstream + [231479] 2023/08/15 01:39:55.564569 [INF] + [231479] 2023/08/15 01:39:55.564572 [INF] ---------------- JETSTREAM ---------------- + [231479] 2023/08/15 01:39:55.564578 [INF] Max Memory: 11.71 GB + [231479] 2023/08/15 01:39:55.564582 [INF] Max Storage: 5.29 GB + [231479] 2023/08/15 01:39:55.564585 [INF] Store Directory: "/tmp/opni-test-2132242568/jetstream/data/jetstream" + [231479] 2023/08/15 01:39:55.564589 [INF] ------------------------------------------- + [231479] 2023/08/15 01:39:55.565067 [INF] Listening for client connections on 0.0.0.0:35793 + [231479] 2023/08/15 01:39:55.565669 [INF] Server is ready + 2023-08-15T01:39:55Z INFO test.env Jetstream started + 2023-08-15T01:39:55Z DEBUG keyring loaded ephemeral key {"path": "/tmp/opni-test-2132242568/keyring/local-agent.json", "usage": "auth", "labels": {"opni.io/session-attribute":"local"}} + 2023-08-15T01:39:55Z DEBUG gateway found 0 plugins + 2023-08-15T01:39:55Z DEBUG gateway loaded plugin manifest {"plugins": 0} + 2023-08-15T01:39:55Z INFO gateway.cache compressing and archiving plugins... + 2023-08-15T01:39:55Z DEBUG gateway.cache added 0 new plugins to cache + 2023-08-15T01:39:55Z INFO gateway running plugin cache gc + 2023-08-15T01:39:55Z INFO gateway.update-server registering update handler for strategy "binary" + 2023-08-15T01:39:55Z INFO gateway.update-server registering update handler for strategy "kubernetes" + 2023-08-15T01:39:55Z INFO gateway.update-server registering update handler for strategy "noop" + 2023-08-15T01:39:55Z DEBUG gateway.grpc registering service {"service": "control.HealthListener"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc registering internal service {"service": "stream.Delegate"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc registering service {"service": "core.Pinger"} + 2023-08-15T01:39:55Z INFO test.env Loading gateway plugins... + 2023-08-15T01:39:55Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.backends.Capability", "opni.Metrics"]} + 2023-08-15T01:39:55Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.Metrics", "opni.backends.Capability", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.System"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.System"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/trace", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/profile", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/block", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/heap", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/ready", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/healthy", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG plugin.metrics/pkg/gateway.stream internal gateway stream connected + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service cortexadmin.CortexAdmin + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/metrics", "capability": "metrics"} + 2023-08-15T01:39:55Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/alerting", "capability": "alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_alerting"} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.AlertConditions + 2023-08-15T01:39:55Z DEBUG plugin.metrics/pkg/gateway.stream calling client handler + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "cortexadmin.CortexAdmin"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/AllUserStats"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/WriteMetrics"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/Query"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/QueryRange"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetRule"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricMetadata"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ListRules"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/LoadRules"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/DeleteRule"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/FlushBlocks"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetSeriesMetrics"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricLabelSets"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexConfig"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ExtractRawSeries"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "cortexadmin.CortexAdmin", "rules": 17} + 2023-08-15T01:39:55Z DEBUG gateway.grpc plugin rejected internal stream connection {"plugin": "plugin_alerting"} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "cortexadmin.CortexAdmin", "rules": [{"http":{"Pattern":{"Get":"/all_user_stats"}},"method":{"name":"AllUserStats","input_type":".google.protobuf.Empty","output_type":".cortexadmin.UserIDStatsList","options":{}}},{"http":{"Pattern":{"Post":"/write_metrics"},"body":"*"},"method":{"name":"WriteMetrics","input_type":".cortexadmin.WriteRequest","output_type":".cortexadmin.WriteResponse","options":{}}},{"http":{"Pattern":{"Get":"/query"},"additional_bindings":[{"Pattern":{"Post":"/query"},"body":"*"}]},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query"},"body":"*"},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/query_range"},"additional_bindings":[{"Pattern":{"Post":"/query_range"},"body":"*"}]},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query_range"},"body":"*"},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/rules/{namespace}/{groupName}"}},"method":{"name":"GetRule","input_type":".cortexadmin.GetRuleRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/metadata"}},"method":{"name":"GetMetricMetadata","input_type":".cortexadmin.MetricMetadataRequest","output_type":".cortexadmin.MetricMetadata","options":{}}},{"http":{"Pattern":{"Get":"/rules"}},"method":{"name":"ListRules","input_type":".cortexadmin.ListRulesRequest","output_type":".cortexadmin.ListRulesResponse","options":{}}},{"http":{"Pattern":{"Post":"/rules"}},"method":{"name":"LoadRules","input_type":".cortexadmin.LoadRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/rules/{groupName}"}},"method":{"name":"DeleteRule","input_type":".cortexadmin.DeleteRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/flush_blocks"}},"method":{"name":"FlushBlocks","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/series/metadata"}},"method":{"name":"GetSeriesMetrics","input_type":".cortexadmin.SeriesRequest","output_type":".cortexadmin.SeriesInfoList","options":{}}},{"http":{"Pattern":{"Get":"/series/labels"}},"method":{"name":"GetMetricLabelSets","input_type":".cortexadmin.LabelRequest","output_type":".cortexadmin.MetricLabels","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetCortexStatus","input_type":".google.protobuf.Empty","output_type":".cortexadmin.CortexStatus","options":{}}},{"http":{"Pattern":{"Get":"/config"}},"method":{"name":"GetCortexConfig","input_type":".cortexadmin.ConfigRequest","output_type":".cortexadmin.ConfigResponse","options":{}}},{"http":{"Pattern":{"Get":"/series/raw"}},"method":{"name":"ExtractRawSeries","input_type":".cortexadmin.MatcherRequest","output_type":".cortexadmin.QueryResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service cortexops.CortexOps + 2023-08-15T01:39:55Z INFO plugin.metrics.cortex-http configuring http api server + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "cortexops.CortexOps"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/ConfigureCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/UninstallCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "cortexops.CortexOps", "rules": 4} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "cortexops.CortexOps", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".cortexops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".cortexops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".cortexops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service remoteread.RemoteReadGateway + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO plugin.alerting using cluster driver {"driver": "test-environment"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /ready", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO plugin.opni.alerting starting opni embedded server {"addr": "127.0.0.1:43869"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "remoteread.RemoteReadGateway"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/AddTarget"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/EditTarget"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/RemoveTarget"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/ListTargets"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Start"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Stop"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/GetTargetStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Discover"} + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.AlertConditions"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionGroups"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CreateAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/GetAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditions"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/UpdateAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionChoices"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeleteAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "remoteread.RemoteReadGateway", "rules": 8} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/AlertConditionStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionsWithStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CloneTo"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ActivateSilence"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeactivateSilence"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/Timeline"} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "remoteread.RemoteReadGateway", "rules": [{"http":{"Pattern":{"Post":"/targets"},"body":"*"},"method":{"name":"AddTarget","input_type":".remoteread.TargetAddRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Put":"/targets"},"body":"*"},"method":{"name":"EditTarget","input_type":".remoteread.TargetEditRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/targets/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"RemoveTarget","input_type":".remoteread.TargetRemoveRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/targets"}},"method":{"name":"ListTargets","input_type":".remoteread.TargetListRequest","output_type":".remoteread.TargetList","options":{}}},{"http":{"Pattern":{"Post":"/targets/start/{target.meta.clusterId}/{target.meta.name}"},"body":"*"},"method":{"name":"Start","input_type":".remoteread.StartReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/targets/stop/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"Stop","input_type":".remoteread.StopReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status"},"body":"*"},"method":{"name":"GetTargetStatus","input_type":".remoteread.TargetStatusRequest","output_type":".remoteread.TargetStatus","options":{}}},{"http":{"Pattern":{"Post":"/edit"},"body":"*"},"method":{"name":"Discover","input_type":".remoteread.DiscoveryRequest","output_type":".remoteread.DiscoveryResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.AlertConditions", "rules": 13} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service node.metrics.NodeConfiguration + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.AlertConditions", "rules": [{"http":{"Pattern":{"Get":"/groups"}},"method":{"name":"ListAlertConditionGroups","input_type":".google.protobuf.Empty","output_type":".core.ReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertCondition","input_type":".alerting.AlertCondition","output_type":".alerting.ConditionReference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertCondition","input_type":".alerting.ConditionReference","output_type":".alerting.AlertCondition","options":{}}},{"http":{"Pattern":{"Post":"/list"},"body":"*"},"method":{"name":"ListAlertConditions","input_type":".alerting.ListAlertConditionRequest","output_type":".alerting.AlertConditionList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertCondition","input_type":".alerting.UpdateAlertConditionRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/choices"}},"method":{"name":"ListAlertConditionChoices","input_type":".alerting.AlertDetailChoicesRequest","output_type":".alerting.ListAlertTypeDetails","options":{}}},{"http":{"Pattern":{"Delete":"/configure"}},"method":{"name":"DeleteAlertCondition","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status/{id}"}},"method":{"name":"AlertConditionStatus","input_type":".alerting.ConditionReference","output_type":".alerting.AlertStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/list/withStatus"},"body":"*"},"method":{"name":"ListAlertConditionsWithStatus","input_type":".alerting.ListStatusRequest","output_type":".alerting.ListStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/clone"},"body":"*"},"method":{"name":"CloneTo","input_type":".alerting.CloneToRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/silences"},"body":"*"},"method":{"name":"ActivateSilence","input_type":".alerting.SilenceRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/silences"}},"method":{"name":"DeactivateSilence","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/timeline"}},"method":{"name":"Timeline","input_type":".alerting.TimelineRequest","output_type":".alerting.TimelineResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.AlertEndpoints + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.AlertEndpoints"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/CreateAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/GetAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ToggleNotifications"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ListAlertEndpoints"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/UpdateAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/DeleteAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/TestAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.AlertEndpoints", "rules": 7} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.AlertEndpoints", "rules": [{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertEndpoint","input_type":".alerting.AlertEndpoint","output_type":".core.Reference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertEndpoint","input_type":".core.Reference","output_type":".alerting.AlertEndpoint","options":{}}},{"http":{"Pattern":{"Post":"/toggleNotifications"},"body":"*"},"method":{"name":"ToggleNotifications","input_type":".alerting.ToggleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/list"}},"method":{"name":"ListAlertEndpoints","input_type":".alerting.ListAlertEndpointsRequest","output_type":".alerting.AlertEndpointList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertEndpoint","input_type":".alerting.UpdateAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/delete"},"body":"*"},"method":{"name":"DeleteAlertEndpoint","input_type":".alerting.DeleteAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/test"},"body":"*"},"method":{"name":"TestAlertEndpoint","input_type":".alerting.TestAlertEndpointRequest","output_type":".alerting.TestAlertEndpointResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.AlertNotifications + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.AlertNotifications"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/TriggerAlerts"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ResolveAlerts"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/PushNotification"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListNotifications"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListAlarmMessages"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListRoutingRelationships"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.AlertNotifications", "rules": 4} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.AlertNotifications", "rules": [{"http":{"Pattern":{"Post":"/trigger"},"body":"*"},"method":{"name":"TriggerAlerts","input_type":".alerting.TriggerAlertsRequest","output_type":".alerting.TriggerAlertsResponse","options":{}}},{"http":{"Pattern":{"Post":"/push"},"body":"*"},"method":{"name":"PushNotification","input_type":".alerting.Notification","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/notifications/list"},"body":"*"},"method":{"name":"ListNotifications","input_type":".alerting.ListNotificationRequest","output_type":".alerting.ListMessageResponse","options":{}}},{"http":{"Pattern":{"Post":"/alarms/list"},"body":"*"},"method":{"name":"ListAlarmMessages","input_type":".alerting.ListAlarmMessageRequest","output_type":".alerting.ListMessageResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.ops.AlertingAdmin + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "node.metrics.NodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "node.metrics.NodeConfiguration", "rules": 4} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "node.metrics.NodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".node.metrics.MetricsCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".node.metrics.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.ops.AlertingAdmin"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/ConfigureCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/InstallCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/UninstallCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/Info"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.ops.AlertingAdmin", "rules": 6} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.ops.AlertingAdmin", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".alerting.ops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".alerting.ops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/install"}},"method":{"name":"InstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".alerting.ops.UninstallRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/info"}},"method":{"name":"Info","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ComponentInfo","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.ops.ConfigReconciler + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.ConfigReconciler/SyncConfig"} + 2023-08-15T01:39:55Z INFO mgmt service has no http rules {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.node.AlertingNodeConfiguration + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.node.AlertingNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.node.AlertingNodeConfiguration", "rules": 4} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.node.AlertingNodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".alerting.node.AlertingCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".alerting.node.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.node.NodeAlertingCapability + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.node.NodeAlertingCapability"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.NodeAlertingCapability/Sync"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.node.NodeAlertingCapability", "rules": 1} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.node.NodeAlertingCapability", "rules": [{"http":{"Pattern":{"Post":"/v1/node/sync"},"body":"*"},"method":{"name":"Sync","input_type":".alerting.node.AlertingCapabilityConfig","output_type":".alerting.node.SyncResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO test.env Waiting for gateway to start... + 2023-08-15T01:39:55Z INFO test.env loaded 12 plugins + 2023-08-15T01:39:55Z INFO mgmt management HTTP server starting {"address": "localhost:39189"} + 2023-08-15T01:39:55Z INFO mgmt management gRPC server starting {"address": "127.0.0.1:42179"} + 2023-08-15T01:39:55Z INFO gateway.grpc gateway gRPC server starting {"address": "127.0.0.1:46029"} + 2023-08-15T01:39:55Z INFO gateway.http gateway HTTP server starting {"api": "127.0.0.1:37611", "metrics": "127.0.0.1:39213"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/tokens"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/tokens/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/watch"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/clusters/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/certs"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}/health"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/health/watch"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/clusters/{cluster.id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/roles"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/roles"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/roles/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/rolebindings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/rolebindings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/subjectaccess"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/apiextensions"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/capabilities"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/capabilities/{name}/installer"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/install"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/uninstall"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/cancel"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/dashboard/settings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/dashboard/settings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/all_user_stats"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/write_metrics"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules/{namespace}/{groupName}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/metadata"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/rules"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/CortexAdmin/rules/{groupName}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/flush_blocks"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/metadata"} + 2023-08-15T01:39:55Z INFO test.env Gateway started + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/labels"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/status"} +[BeforeSuite] PASSED [0.234 seconds] +------------------------------ + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/raw"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/configuration"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/uninstall"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/RemoteReadGateway/targets"} +agent capability tests when we use the alerting downstream capability should install the alerting cluster [integration] +/home/ec2-user/opni/test/plugins/alerting/capability_test.go:37 + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/RemoteReadGateway/targets/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:39:55Z INFO test.env Starting test environment + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/start/{target.meta.clusterId}/{target.meta.name}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/stop/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/edit"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertConditions/groups"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertConditions/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/choices"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/status/{id}"} + 2023-08-15T01:39:55Z INFO test.env Copying 7 files from embedded testdata/cortex to /tmp/opni-test-278561545/cortex + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/withStatus"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/clone"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/silences"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/silences"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/timeline"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/list/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/toggleNotifications"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertEndpoints/list"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/delete"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/test"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/trigger"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/push"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/notifications/list"} + 2023-08-15T01:39:55Z INFO test.env Waiting for jetstream to start... + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/alarms/list"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/configuration"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/install"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/uninstall"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/info"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/NodeAlertingCapability/v1/node/sync"} + [231489] 2023/08/15 01:39:55.799753 [INF] Starting nats-server + [231489] 2023/08/15 01:39:55.799817 [INF] Version: 2.9.21 + [231489] 2023/08/15 01:39:55.799822 [INF] Git: [b2e7725] + [231489] 2023/08/15 01:39:55.799834 [INF] Name: NDOMIA7P3BPCZ4AKM2VQ4E5SHQIQG7KFFF7UJKGLLKOMSPOHHVF362XI + [231489] 2023/08/15 01:39:55.799840 [INF] Node: 3KjorgZ4 + [231489] 2023/08/15 01:39:55.799843 [INF] ID: NDOMIA7P3BPCZ4AKM2VQ4E5SHQIQG7KFFF7UJKGLLKOMSPOHHVF362XI + [231489] 2023/08/15 01:39:55.799862 [INF] Using configuration file: /tmp/opni-test-278561545/jetstream/jetstream.conf + [231489] 2023/08/15 01:39:55.800154 [INF] Starting JetStream + [231489] 2023/08/15 01:39:55.800348 [INF] _ ___ _____ ___ _____ ___ ___ _ __ __ + [231489] 2023/08/15 01:39:55.800366 [INF] _ | | __|_ _/ __|_ _| _ \ __| /_\ | \/ | + [231489] 2023/08/15 01:39:55.800370 [INF] | || | _| | | \__ \ | | | / _| / _ \| |\/| | + [231489] 2023/08/15 01:39:55.800372 [INF] \__/|___| |_| |___/ |_| |_|_\___/_/ \_\_| |_| + [231489] 2023/08/15 01:39:55.800376 [INF] + [231489] 2023/08/15 01:39:55.800380 [INF] https://docs.nats.io/jetstream + [231489] 2023/08/15 01:39:55.800382 [INF] + [231489] 2023/08/15 01:39:55.800392 [INF] ---------------- JETSTREAM ---------------- + [231489] 2023/08/15 01:39:55.800399 [INF] Max Memory: 11.71 GB + [231489] 2023/08/15 01:39:55.800410 [INF] Max Storage: 5.29 GB + [231489] 2023/08/15 01:39:55.800419 [INF] Store Directory: "/tmp/opni-test-278561545/jetstream/data/jetstream" + [231489] 2023/08/15 01:39:55.800430 [INF] ------------------------------------------- + [231489] 2023/08/15 01:39:55.800894 [INF] Listening for client connections on 0.0.0.0:32947 + [231489] 2023/08/15 01:39:55.801237 [INF] Server is ready + 2023-08-15T01:39:55Z INFO test.env Jetstream started + 2023-08-15T01:39:55Z DEBUG keyring loaded ephemeral key {"path": "/tmp/opni-test-278561545/keyring/local-agent.json", "usage": "auth", "labels": {"opni.io/session-attribute":"local"}} + 2023-08-15T01:39:55Z DEBUG gateway found 0 plugins + 2023-08-15T01:39:55Z DEBUG gateway loaded plugin manifest {"plugins": 0} + 2023-08-15T01:39:55Z INFO gateway.cache compressing and archiving plugins... + 2023-08-15T01:39:55Z DEBUG gateway.cache added 0 new plugins to cache + 2023-08-15T01:39:55Z INFO gateway running plugin cache gc + 2023-08-15T01:39:55Z INFO gateway.update-server registering update handler for strategy "binary" + 2023-08-15T01:39:55Z INFO gateway.update-server registering update handler for strategy "kubernetes" + 2023-08-15T01:39:55Z INFO gateway.update-server registering update handler for strategy "noop" + 2023-08-15T01:39:55Z DEBUG gateway.grpc registering service {"service": "control.HealthListener"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc registering internal service {"service": "stream.Delegate"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc registering service {"service": "core.Pinger"} + 2023-08-15T01:39:55Z INFO test.env Loading gateway plugins... + 2023-08-15T01:39:55Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.Metrics", "opni.backends.Capability", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:39:55Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.Metrics", "opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.backends.Capability"]} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.System"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.System"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_metrics"} + 2023-08-15T01:39:55Z DEBUG plugin.metrics/pkg/gateway.stream internal gateway stream connected + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.AlertConditions + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/trace", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/profile", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/block", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/heap", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/ready", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/healthy", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:55Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/alerting", "capability": "alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_alerting"} + 2023-08-15T01:39:55Z DEBUG gateway.grpc plugin rejected internal stream connection {"plugin": "plugin_alerting"} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service cortexadmin.CortexAdmin + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG plugin.metrics/pkg/gateway.stream calling client handler + 2023-08-15T01:39:55Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/metrics", "capability": "metrics"} + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.AlertConditions"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionGroups"} + 2023-08-15T01:39:55Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Metrics"} + 2023-08-15T01:39:55Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CreateAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/GetAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditions"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/UpdateAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionChoices"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeleteAlertCondition"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/AlertConditionStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionsWithStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CloneTo"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ActivateSilence"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeactivateSilence"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertConditions/Timeline"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.AlertConditions", "rules": 13} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.AlertConditions", "rules": [{"http":{"Pattern":{"Get":"/groups"}},"method":{"name":"ListAlertConditionGroups","input_type":".google.protobuf.Empty","output_type":".core.ReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertCondition","input_type":".alerting.AlertCondition","output_type":".alerting.ConditionReference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertCondition","input_type":".alerting.ConditionReference","output_type":".alerting.AlertCondition","options":{}}},{"http":{"Pattern":{"Post":"/list"},"body":"*"},"method":{"name":"ListAlertConditions","input_type":".alerting.ListAlertConditionRequest","output_type":".alerting.AlertConditionList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertCondition","input_type":".alerting.UpdateAlertConditionRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/choices"}},"method":{"name":"ListAlertConditionChoices","input_type":".alerting.AlertDetailChoicesRequest","output_type":".alerting.ListAlertTypeDetails","options":{}}},{"http":{"Pattern":{"Delete":"/configure"}},"method":{"name":"DeleteAlertCondition","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status/{id}"}},"method":{"name":"AlertConditionStatus","input_type":".alerting.ConditionReference","output_type":".alerting.AlertStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/list/withStatus"},"body":"*"},"method":{"name":"ListAlertConditionsWithStatus","input_type":".alerting.ListStatusRequest","output_type":".alerting.ListStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/clone"},"body":"*"},"method":{"name":"CloneTo","input_type":".alerting.CloneToRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/silences"},"body":"*"},"method":{"name":"ActivateSilence","input_type":".alerting.SilenceRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/silences"}},"method":{"name":"DeactivateSilence","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/timeline"}},"method":{"name":"Timeline","input_type":".alerting.TimelineRequest","output_type":".alerting.TimelineResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.AlertEndpoints + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.AlertEndpoints"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/CreateAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/GetAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ToggleNotifications"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ListAlertEndpoints"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/UpdateAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/DeleteAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/TestAlertEndpoint"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.AlertEndpoints", "rules": 7} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.AlertEndpoints", "rules": [{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertEndpoint","input_type":".alerting.AlertEndpoint","output_type":".core.Reference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertEndpoint","input_type":".core.Reference","output_type":".alerting.AlertEndpoint","options":{}}},{"http":{"Pattern":{"Post":"/toggleNotifications"},"body":"*"},"method":{"name":"ToggleNotifications","input_type":".alerting.ToggleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/list"}},"method":{"name":"ListAlertEndpoints","input_type":".alerting.ListAlertEndpointsRequest","output_type":".alerting.AlertEndpointList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertEndpoint","input_type":".alerting.UpdateAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/delete"},"body":"*"},"method":{"name":"DeleteAlertEndpoint","input_type":".alerting.DeleteAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/test"},"body":"*"},"method":{"name":"TestAlertEndpoint","input_type":".alerting.TestAlertEndpointRequest","output_type":".alerting.TestAlertEndpointResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.AlertNotifications + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.AlertNotifications"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/TriggerAlerts"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ResolveAlerts"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/PushNotification"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListNotifications"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListAlarmMessages"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListRoutingRelationships"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.AlertNotifications", "rules": 4} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.AlertNotifications", "rules": [{"http":{"Pattern":{"Post":"/trigger"},"body":"*"},"method":{"name":"TriggerAlerts","input_type":".alerting.TriggerAlertsRequest","output_type":".alerting.TriggerAlertsResponse","options":{}}},{"http":{"Pattern":{"Post":"/push"},"body":"*"},"method":{"name":"PushNotification","input_type":".alerting.Notification","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/notifications/list"},"body":"*"},"method":{"name":"ListNotifications","input_type":".alerting.ListNotificationRequest","output_type":".alerting.ListMessageResponse","options":{}}},{"http":{"Pattern":{"Post":"/alarms/list"},"body":"*"},"method":{"name":"ListAlarmMessages","input_type":".alerting.ListAlarmMessageRequest","output_type":".alerting.ListMessageResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.ops.AlertingAdmin + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "cortexadmin.CortexAdmin"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/AllUserStats"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/WriteMetrics"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/Query"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/QueryRange"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetRule"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricMetadata"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ListRules"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/LoadRules"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/DeleteRule"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/FlushBlocks"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetSeriesMetrics"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricLabelSets"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexConfig"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ExtractRawSeries"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "cortexadmin.CortexAdmin", "rules": 17} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "cortexadmin.CortexAdmin", "rules": [{"http":{"Pattern":{"Get":"/all_user_stats"}},"method":{"name":"AllUserStats","input_type":".google.protobuf.Empty","output_type":".cortexadmin.UserIDStatsList","options":{}}},{"http":{"Pattern":{"Post":"/write_metrics"},"body":"*"},"method":{"name":"WriteMetrics","input_type":".cortexadmin.WriteRequest","output_type":".cortexadmin.WriteResponse","options":{}}},{"http":{"Pattern":{"Get":"/query"},"additional_bindings":[{"Pattern":{"Post":"/query"},"body":"*"}]},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query"},"body":"*"},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/query_range"},"additional_bindings":[{"Pattern":{"Post":"/query_range"},"body":"*"}]},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query_range"},"body":"*"},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/rules/{namespace}/{groupName}"}},"method":{"name":"GetRule","input_type":".cortexadmin.GetRuleRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/metadata"}},"method":{"name":"GetMetricMetadata","input_type":".cortexadmin.MetricMetadataRequest","output_type":".cortexadmin.MetricMetadata","options":{}}},{"http":{"Pattern":{"Get":"/rules"}},"method":{"name":"ListRules","input_type":".cortexadmin.ListRulesRequest","output_type":".cortexadmin.ListRulesResponse","options":{}}},{"http":{"Pattern":{"Post":"/rules"}},"method":{"name":"LoadRules","input_type":".cortexadmin.LoadRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/rules/{groupName}"}},"method":{"name":"DeleteRule","input_type":".cortexadmin.DeleteRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/flush_blocks"}},"method":{"name":"FlushBlocks","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/series/metadata"}},"method":{"name":"GetSeriesMetrics","input_type":".cortexadmin.SeriesRequest","output_type":".cortexadmin.SeriesInfoList","options":{}}},{"http":{"Pattern":{"Get":"/series/labels"}},"method":{"name":"GetMetricLabelSets","input_type":".cortexadmin.LabelRequest","output_type":".cortexadmin.MetricLabels","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetCortexStatus","input_type":".google.protobuf.Empty","output_type":".cortexadmin.CortexStatus","options":{}}},{"http":{"Pattern":{"Get":"/config"}},"method":{"name":"GetCortexConfig","input_type":".cortexadmin.ConfigRequest","output_type":".cortexadmin.ConfigResponse","options":{}}},{"http":{"Pattern":{"Get":"/series/raw"}},"method":{"name":"ExtractRawSeries","input_type":".cortexadmin.MatcherRequest","output_type":".cortexadmin.QueryResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service cortexops.CortexOps + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.ops.AlertingAdmin"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/ConfigureCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/InstallCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/UninstallCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/Info"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.ops.AlertingAdmin", "rules": 6} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.ops.AlertingAdmin", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".alerting.ops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".alerting.ops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/install"}},"method":{"name":"InstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".alerting.ops.UninstallRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/info"}},"method":{"name":"Info","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ComponentInfo","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.ops.ConfigReconciler + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.ops.ConfigReconciler/SyncConfig"} + 2023-08-15T01:39:55Z INFO mgmt service has no http rules {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:39:55Z INFO plugin.alerting using cluster driver {"driver": "test-environment"} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.node.AlertingNodeConfiguration + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.node.AlertingNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.node.AlertingNodeConfiguration", "rules": 4} + 2023-08-15T01:39:55Z INFO plugin.opni.alerting starting opni embedded server {"addr": "127.0.0.1:35689"} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.node.AlertingNodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".alerting.node.AlertingCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".alerting.node.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service alerting.node.NodeAlertingCapability + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "alerting.node.NodeAlertingCapability"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/alerting.node.NodeAlertingCapability/Sync"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "alerting.node.NodeAlertingCapability", "rules": 1} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "alerting.node.NodeAlertingCapability", "rules": [{"http":{"Pattern":{"Post":"/v1/node/sync"},"body":"*"},"method":{"name":"Sync","input_type":".alerting.node.AlertingCapabilityConfig","output_type":".alerting.node.SyncResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO plugin.metrics.cortex-http configuring http api server + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "cortexops.CortexOps"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/ConfigureCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/cortexops.CortexOps/UninstallCluster"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "cortexops.CortexOps", "rules": 4} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "cortexops.CortexOps", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".cortexops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".cortexops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".cortexops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service remoteread.RemoteReadGateway + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /ready", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "GET /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PUT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "PATCH /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "HEAD /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "DELETE /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z DEBUG gateway.http configured route for plugin {"route": "TRACE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "remoteread.RemoteReadGateway"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/AddTarget"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/EditTarget"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/RemoveTarget"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/ListTargets"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Start"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Stop"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/GetTargetStatus"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Discover"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "remoteread.RemoteReadGateway", "rules": 8} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "remoteread.RemoteReadGateway", "rules": [{"http":{"Pattern":{"Post":"/targets"},"body":"*"},"method":{"name":"AddTarget","input_type":".remoteread.TargetAddRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Put":"/targets"},"body":"*"},"method":{"name":"EditTarget","input_type":".remoteread.TargetEditRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/targets/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"RemoveTarget","input_type":".remoteread.TargetRemoveRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/targets"}},"method":{"name":"ListTargets","input_type":".remoteread.TargetListRequest","output_type":".remoteread.TargetList","options":{}}},{"http":{"Pattern":{"Post":"/targets/start/{target.meta.clusterId}/{target.meta.name}"},"body":"*"},"method":{"name":"Start","input_type":".remoteread.StartReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/targets/stop/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"Stop","input_type":".remoteread.StopReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status"},"body":"*"},"method":{"name":"GetTargetStatus","input_type":".remoteread.TargetStatusRequest","output_type":".remoteread.TargetStatus","options":{}}},{"http":{"Pattern":{"Post":"/edit"},"body":"*"},"method":{"name":"Discover","input_type":".remoteread.DiscoveryRequest","output_type":".remoteread.DiscoveryResponse","options":{}}}]} + 2023-08-15T01:39:55Z INFO mgmt got extension descriptor for service node.metrics.NodeConfiguration + 2023-08-15T01:39:55Z INFO mgmt loading service {"name": "node.metrics.NodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:39:55Z INFO mgmt loading http rules {"name": "node.metrics.NodeConfiguration", "rules": 4} + 2023-08-15T01:39:55Z DEBUG mgmt rule descriptors {"name": "node.metrics.NodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".node.metrics.MetricsCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".node.metrics.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:39:55Z INFO test.env Waiting for gateway to start... + 2023-08-15T01:39:55Z INFO test.env loaded 12 plugins + 2023-08-15T01:39:55Z INFO gateway.grpc gateway gRPC server starting {"address": "127.0.0.1:38891"} + 2023-08-15T01:39:55Z INFO gateway.http gateway HTTP server starting {"api": "127.0.0.1:43103", "metrics": "127.0.0.1:36995"} + 2023-08-15T01:39:55Z INFO mgmt management HTTP server starting {"address": "localhost:34163"} + 2023-08-15T01:39:55Z INFO mgmt management gRPC server starting {"address": "127.0.0.1:46361"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/tokens"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/tokens/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/watch"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/clusters/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/certs"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}/health"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/health/watch"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/clusters/{cluster.id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/roles"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/roles"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/roles/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/rolebindings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/rolebindings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/subjectaccess"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/apiextensions"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/capabilities"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/capabilities/{name}/installer"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/install"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/uninstall"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/cancel"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/dashboard/settings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/dashboard/settings"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertConditions/groups"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertConditions/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/choices"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/status/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/withStatus"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/clone"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/silences"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/silences"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/timeline"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/list/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/toggleNotifications"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertEndpoints/list"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/delete"} + 2023-08-15T01:39:55Z INFO test.env Gateway started + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/test"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/trigger"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/push"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/notifications/list"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/alarms/list"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/all_user_stats"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/write_metrics"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules/{namespace}/{groupName}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/metadata"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/rules"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/CortexAdmin/rules/{groupName}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/flush_blocks"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/metadata"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/labels"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/raw"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/configuration"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/install"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/uninstall"} + 2023-08-15T01:39:55Z DEBUG plugin.alerting-test-cluster-driver Syncer start : alerting-server --syncer.alertmanager.config.file=/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml --syncer.listen.address=:45391 --syncer.alertmanager.address=http://127.0.0.1:33771 --syncer.gateway.join.address=:46361 syncer {"config-file": "/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/info"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:39:55Z INFO plugin.alerting-test-cluster-driver Starting AlertManager {"config-file": "/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml", "alertmanager-port": 33771, "opni-port": 41223} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/NodeAlertingCapability/v1/node/sync"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/configuration"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/configure"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/uninstall"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/RemoteReadGateway/targets/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/start/{target.meta.clusterId}/{target.meta.name}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/stop/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/status"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/edit"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config/{id}"} + 2023-08-15T01:39:55Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config/{node.id}"} + ts=2023-08-15T01:39:56.082Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:39:56.082Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:39:56.085Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=44749 + ts=2023-08-15T01:39:56.086Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:39:56.116Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml + ts=2023-08-15T01:39:56.117Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml + ts=2023-08-15T01:39:56.119Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:33771 + ts=2023-08-15T01:39:56.119Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:33771 + 2023-08-15T01:39:56Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:39:56Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:39:56Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:39:56Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:39:56Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:39:56Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:39:56Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:39:56Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:39:56Z INFO plugin.alerting-test-cluster-driver Starting AlertManager Syncer {"config-file": "/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml", "port": 45391} + 2023-08-15T01:39:56Z INFO plugin.alerting-test-cluster-driver AlertManager started {"config-file": "/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml", "address": "http://127.0.0.1:33771"} + 2023-08-15T01:39:56Z INFO plugin.alerting updating known peers : [{http://127.0.0.1:33771 http://127.0.0.1:35689}] +• [1.209 seconds] +------------------------------ +agent capability tests when we use the alerting downstream capability should be able to install the alerting capability on all downstream agents [integration] +/home/ec2-user/opni/test/plugins/alerting/capability_test.go:60 + STEP: bootstrapping agents @ 08/15/23 01:39:56.998 + 2023-08-15T01:39:57Z INFO test starting agent {"id": "agent1", "address": "127.0.0.1:43003", "version": "v2"} + 2023-08-15T01:39:57Z DEBUG agent using log level: debug + 2023-08-15T01:39:57Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:39:57Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent1 + 2023-08-15T01:39:57Z INFO agent bootstrap completed successfully + 2023-08-15T01:39:57Z WARN jetstream disconnected from jetstream + 2023-08-15T01:39:57Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:39:57Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:39:57Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:39:57Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG commands/alerting.go:72 syncer gateway join address:46361 + 2023-08-15T01:39:57Z INFO plugin.alerting-syncer syncer/syncer_server.go:90 starting alerting syncer server as identity 2322db02-3463-4d35-bdf7-078193fc2f5e + 2023-08-15T01:39:57Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.apiextensions.HTTPAPIExtension", "opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:108 acquired gateway alerting client + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:144 starting alerting syncer stream subscription + 2023-08-15T01:39:57Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:39:57Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "4d74f48a-1c76-4e7f-a440-257aa59e8ac1"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:150 starting new stream subscription + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:39:57Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "447ca987-ca2d-4c53-868b-50730809d173"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:39:57Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:39:57Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:39:57Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:39:57Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:39:57Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:39:57Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:39:57Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:39:57Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:39:57Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:39:57Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:39:57Z INFO agent using unmanaged plugin loader + 2023-08-15T01:39:57Z INFO agent agent http server starting {"address": "127.0.0.1:43003"} + 2023-08-15T01:39:57Z INFO agent connecting to gateway... + 2023-08-15T01:39:57Z DEBUG gateway ratelimit: 50.000000 available + 2023-08-15T01:39:57Z INFO test starting agent {"id": "agent2", "address": "127.0.0.1:33707", "version": "v2"} + 2023-08-15T01:39:57Z DEBUG agent using log level: debug + 2023-08-15T01:39:57Z DEBUG gateway-client authenticated {"id": "agent1"} + 2023-08-15T01:39:57Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml"} + 2023-08-15T01:39:57Z DEBUG gateway.grpc handling new stream connection + ts=2023-08-15T01:39:57.110Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml + ts=2023-08-15T01:39:57.111Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:39:57Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-278561545/alertmanager-config-058b2ac2-d73e-46fa-b7f7-58a6e2bec07d/alertmanager.yaml"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent1"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:39:57Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent1"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:39:57Z DEBUG gateway.sync agent connected {"id": "agent1"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:39:57Z INFO gateway.monitor received status update {"id": "agent1", "connected": true} + 2023-08-15T01:39:57Z INFO agent bootstrap completed successfully + 2023-08-15T01:39:57Z DEBUG gateway.delegate agent connected {"id": "agent1"} + 2023-08-15T01:39:57Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent2 + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:39:57Z WARN jetstream disconnected from jetstream + 2023-08-15T01:39:57Z INFO agent gateway connected + 2023-08-15T01:39:57Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:39:57Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:39:57Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z INFO gateway.monitor received health update {"id": "agent1", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:39:57Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent1", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:39:57Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:39:57Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:39:57Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:39:57Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:39:57Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:39:57Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:39:57Z INFO gateway.monitor received health update {"id": "agent1", "ready": true, "conditions": []} + 2023-08-15T01:39:57Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:39:57Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:39:57Z DEBUG test.env configuring node {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:39:57Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:39:57Z DEBUG test.env configuring node {"node": "agent1", "driver": "otel"} + 2023-08-15T01:39:57Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "otel"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent1"} + 2023-08-15T01:39:57Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:39:57Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:39:57Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:39:57Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:39:57Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:39:57Z INFO agent using unmanaged plugin loader + 2023-08-15T01:39:57Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:39:57Z INFO agent agent http server starting {"address": "127.0.0.1:33707"} + 2023-08-15T01:39:57Z INFO agent connecting to gateway... + 2023-08-15T01:39:57Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:39:57Z DEBUG gateway ratelimit: 49.952639 available + 2023-08-15T01:39:57Z INFO test starting agent {"id": "agent3", "address": "127.0.0.1:40781", "version": "v2"} + 2023-08-15T01:39:57Z DEBUG gateway-client authenticated {"id": "agent2"} + 2023-08-15T01:39:57Z DEBUG agent using log level: debug + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:39:57Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:39:57Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:39:57Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG gateway.sync agent connected {"id": "agent2"} + 2023-08-15T01:39:57Z INFO gateway.monitor received status update {"id": "agent2", "connected": true} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:39:57Z DEBUG gateway.delegate agent connected {"id": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:39:57Z INFO gateway.monitor received health update {"id": "agent2", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:39:57Z INFO agent gateway connected + 2023-08-15T01:39:57Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent3 + 2023-08-15T01:39:57Z INFO agent bootstrap completed successfully + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:39:57Z WARN jetstream disconnected from jetstream + 2023-08-15T01:39:57Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:39:57Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:39:57Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent3 is now active + 2023-08-15T01:39:57Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:39:57Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent2", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent3 is now active + 2023-08-15T01:39:57Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:39:57Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:39:57Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:39:57Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent2"} + 2023-08-15T01:39:57Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:39:57Z DEBUG test.env configuring node {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:39:57Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:39:57Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:39:57Z DEBUG test.env configuring node {"node": "agent2", "driver": "otel"} + 2023-08-15T01:39:57Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "otel"} + 2023-08-15T01:39:57Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:39:57Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:39:57Z INFO gateway.monitor received health update {"id": "agent2", "ready": true, "conditions": []} + 2023-08-15T01:39:57Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:39:57Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:39:57Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:39:57Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:39:57Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:39:57Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:39:57Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:39:57Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:39:57Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:39:57Z INFO agent using unmanaged plugin loader + 2023-08-15T01:39:57Z INFO agent agent http server starting {"address": "127.0.0.1:40781"} + 2023-08-15T01:39:57Z INFO agent connecting to gateway... + 2023-08-15T01:39:57Z DEBUG gateway ratelimit: 49.942679 available + 2023-08-15T01:39:57Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["alerting"]} + 2023-08-15T01:39:57Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG gateway-client authenticated {"id": "agent3"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:39:57Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent1", "capability": "alerting"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent2", "capabilities": ["alerting"]} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent3"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:39:57Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent2", "capability": "alerting"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent3"} + 2023-08-15T01:39:57Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting starting rule stream {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:39:57Z INFO plugin.alerting waiting for rule sync client... {"component": "rule-streamer"} + 2023-08-15T01:39:57Z INFO plugin.alerting rule sync client acquired {"component": "rule-streamer"} + 2023-08-15T01:39:57Z INFO plugin.alerting discovered 1 rules {"component": "rule-streamer"} + 2023-08-15T01:39:57Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent3", "capability": "alerting"} + 2023-08-15T01:39:57Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting starting rule stream {"component": "rule-streamer", "nodeId": "agent2"} +• [0.332 seconds] +------------------------------ + 2023-08-15T01:39:57Z INFO plugin.alerting waiting for rule sync client... {"component": "rule-streamer"} + 2023-08-15T01:39:57Z INFO plugin.alerting rule sync client acquired {"component": "rule-streamer"} + 2023-08-15T01:39:57Z INFO plugin.alerting discovered 1 rules {"component": "rule-streamer"} +agent capability tests when we use the alerting downstream capability should keep track of the downstream synced alerting groups [integration] +/home/ec2-user/opni/test/plugins/alerting/capability_test.go:92 + 2023-08-15T01:39:57Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:39:57Z DEBUG gateway.sync agent connected {"id": "agent3"} + 2023-08-15T01:39:57Z INFO gateway.monitor received status update {"id": "agent3", "connected": true} + 2023-08-15T01:39:57Z DEBUG gateway.delegate agent connected {"id": "agent3"} + 2023-08-15T01:39:57Z INFO gateway.monitor received health update {"id": "agent3", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:39:57Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z INFO agent gateway connected + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent3", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:39:57Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent3"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting starting rule stream {"component": "rule-streamer", "nodeId": "agent3"} + 2023-08-15T01:39:57Z INFO gateway.monitor received health update {"id": "agent3", "ready": true, "conditions": []} + 2023-08-15T01:39:57Z INFO plugin.alerting waiting for rule sync client... {"component": "rule-streamer"} + 2023-08-15T01:39:57Z INFO plugin.alerting rule sync client acquired {"component": "rule-streamer"} + 2023-08-15T01:39:57Z INFO plugin.alerting discovered 1 rules {"component": "rule-streamer"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:39:57Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:39:57Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:39:57Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent3", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:39:57Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:39:57Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent3"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:39:57Z DEBUG test.env configuring node {"node": "agent3", "driver": "otel"} + 2023-08-15T01:39:57Z DEBUG test.env nothing to do (already stopped) {"node": "agent3", "driver": "otel"} + 2023-08-15T01:39:57Z DEBUG test.env configuring node {"node": "agent3", "driver": "prometheus"} + 2023-08-15T01:39:57Z DEBUG test.env nothing to do (already stopped) {"node": "agent3", "driver": "prometheus"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:39:57Z DEBUG plugin.metrics sent node health update +• [0.214 seconds] +------------------------------ +agent capability tests when we use the alerting downstream capability should have synced downstream prometheus alerting rules [integration] +/home/ec2-user/opni/test/plugins/alerting/capability_test.go:107 +• [0.033 seconds] +------------------------------ +agent capability tests when we use the alerting downstream capability should be able to manipulate a sub-set of the synced rule configuration [integration] +/home/ec2-user/opni/test/plugins/alerting/capability_test.go:141 + 2023-08-15T01:39:57Z DEBUG plugin.alerting Updating alert condition id:"test-rule-16578271-d08c-4b36-b12e-e8026220a9b3" groupId:"test-group-16578271-d08c-4b36-b12e-e8026220a9b3" {"component": "alarms", "handler": "UpdateAlertCondition"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting Updating alert condition id:"test-rule-944f7e8d-6a94-4f4d-92a4-55b77d724d8a" groupId:"test-group-944f7e8d-6a94-4f4d-92a4-55b77d724d8a" {"component": "alarms", "handler": "UpdateAlertCondition"} + 2023-08-15T01:39:57Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting Updating alert condition id:"test-rule-eb475fbb-4d41-48d1-8a4c-5ceca3228273" groupId:"test-group-eb475fbb-4d41-48d1-8a4c-5ceca3228273" {"component": "alarms", "handler": "UpdateAlertCondition"} + 2023-08-15T01:39:57Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:39:57Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:39:57Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} +• [0.042 seconds] +------------------------------ +agent capability tests when we use the alerting downstream capability should be able to uninstall the alerting capability on all downstream agents [integration] +/home/ec2-user/opni/test/plugins/alerting/capability_test.go:186 + 2023-08-15T01:39:57Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["alerting"]} + 2023-08-15T01:39:57Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:39:57Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent1", "capability": "alerting"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent2", "capabilities": ["alerting"]} + 2023-08-15T01:39:57Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting stopping rule stream {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:39:57Z INFO plugin.alerting Exiting rule sync loop {"component": "rule-streamer"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:39:57Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent2", "capability": "alerting"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent3", "capabilities": ["alerting"]} + 2023-08-15T01:39:57Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting stopping rule stream {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z INFO plugin.alerting Exiting rule sync loop {"component": "rule-streamer"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:39:57Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:39:57Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent3", "capability": "alerting"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent3", "time": "2023-08-15T01:39:57Z"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "447ca987-ca2d-4c53-868b-50730809d173"} + 2023-08-15T01:39:57Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "4d74f48a-1c76-4e7f-a440-257aa59e8ac1"} + 2023-08-15T01:39:57Z INFO plugin.alerting shutting down cluster driver update handler + 2023-08-15T01:39:57Z INFO plugin.alerting exiting main sync loop + 2023-08-15T01:39:57Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:39:57Z INFO plugin.alerting condition set {"condition": "Config Sync", "status": "Failure", "reason": "error syncing alerting node : context canceled"} + 2023-08-15T01:39:57Z ERROR test agent "agent1" exited: context canceled + 2023-08-15T01:39:57Z DEBUG plugin.alerting closing cortex cluster status watcher... {"watcher": "cortex-cluster-status"} + 2023-08-15T01:39:57Z ERROR plugin.alerting failed to receive cluster event : rpc error: code = Canceled desc = context canceled + 2023-08-15T01:39:57Z WARN plugin.alerting failed to send %s node health updatesalerting {"component": "health-cfg-sync", "err": "context canceled"} + 2023-08-15T01:39:57Z WARN jetstream disconnected from jetstream + 2023-08-15T01:39:57Z INFO gateway shutting down plugins + 2023-08-15T01:39:57Z INFO plugin.alerting Exiting rule sync loop {"component": "rule-streamer"} + 2023-08-15T01:39:57Z WARN plugin.alerting failed to receive cluster health status from grpc stream, retrying... +2023-08-15T01:39:57.654Z WARN totem.gateway-server stream.go:231 stream error; kicking {"error": "rpc error: code = Unavailable desc = transport is closing"} + 2023-08-15T01:39:57Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN jetstream disconnected from jetstream + 2023-08-15T01:39:57Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:39:57Z DEBUG gateway.delegate agent disconnected {"id": "agent1"} + 2023-08-15T01:39:57Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:39:57Z DEBUG gateway.sync agent disconnected {"id": "agent1"} + 2023-08-15T01:39:57.655Z WARN totem.gateway-server server.go:222 stream exited with error {"error": "rpc error: code = Unavailable desc = transport is closing"} +2023-08-15T01:39:57Z INFO gateway all plugins shut down + 2023-08-15T01:39:57Z WARN jetstream disconnected from jetstream + 2023-08-15T01:39:57Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:39:57Z DEBUG gateway.delegate agent disconnected {"id": "agent2"} + 2023-08-15T01:39:57Z DEBUG gateway.sync agent disconnected {"id": "agent2"} + 2023-08-15T01:39:57Z INFO gateway.grpc agent stream closing {"error": "context canceled"} +2023-08-15T01:39:57.655Z WARN totem.gateway-server.gateway-server->plugin_metrics.gateway-server->plugin_metrics stream.go:231 stream error; kicking {"error": "rpc error: code = Unavailable desc = transport is closing"} + 2023-08-15T01:39:57Z WARN gateway http server exited with error {"error": "context canceled"} +2023-08-15T01:39:57.657Z WARN totem.gateway-server server.go:154 stream exited with error {"error": "rpc error: code = Unavailable desc = transport is closing"} + 2023-08-15T01:39:57Z ERROR test agent "agent3" exited: context canceled + 2023-08-15T01:39:57Z ERROR test agent "agent2" exited: context canceled + 2023-08-15T01:39:57Z WARN test.env management server exited with error {"error": "context canceled"} + 2023-08-15T01:39:57Z DEBUG gateway.delegate agent disconnected {"id": "agent3"} + 2023-08-15T01:39:57Z WARN test.env gateway server exited with error {"error": "context canceled"} + 2023-08-15T01:39:57Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:39:57Z DEBUG gateway.sync agent disconnected {"id": "agent3"} + 2023-08-15T01:39:57Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} +2023-08-15T01:39:57.658Z WARN totem.gateway-server.gateway-server->plugin_alerting.gateway-server->plugin_alerting stream.go:231 stream error; kicking {"error": "rpc error: code = Unavailable desc = transport is closing"} + ts=2023-08-15T01:39:57.651Z caller=alertmanager_main.go:601 level=info msg="Received SIGTERM, exiting gracefully..." + ts=2023-08-15T01:39:57.652Z caller=cluster.go:690 level=info component=cluster msg="gossip not settled but continuing anyway" polls=0 elapsed=1.565609031s + 2023-08-15T01:39:57Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error + 2023-08-15T01:39:57Z ERROR plugin.alerting EOF + 2023-08-15T01:39:57Z ERROR plugin.alerting failed to get node capability spec {"component": "node-backend", "error": "rpc error: code = Unknown desc = nats: connection closed"} + 2023-08-15T01:39:57Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error + 2023-08-15T01:39:57Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:39:57Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error + 2023-08-15T01:39:57Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error + 2023-08-15T01:39:57Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} +• [0.044 seconds] +------------------------------ +Alerting routing logic translation to physical dispatching when setting namespace specs on the routing tree should be able to dynamically update alert routing [integration] +/home/ec2-user/opni/test/plugins/alerting/routing_test.go:77 + STEP: Creating some test webhook servers @ 08/15/23 01:39:57.667 + STEP: routing to a subset of the test webhook servers @ 08/15/23 01:39:57.668 + STEP: setting the router to the namespace specs @ 08/15/23 01:39:57.668 + STEP: running alertmanager with this config @ 08/15/23 01:39:57.668 + STEP: expecting to successfully build a config @ 08/15/23 01:39:57.669 + STEP: checking that the config passes the marshalling validation @ 08/15/23 01:39:57.669 + STEP: checking that the config passes the unmarshalling validation @ 08/15/23 01:39:57.673 + STEP: writing the config file to a temporary directory @ 08/15/23 01:39:57.678 + STEP: Verifying that the config can be loaded by alertmanager @ 08/15/23 01:39:57.679 + 2023-08-15T01:39:57Z INFO test.env Waiting for alertmanager to start... + ts=2023-08-15T01:39:57.778Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:39:57.778Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:39:57.778Z caller=cluster.go:173 level=debug component=cluster msg="resolved peers to following addresses" peers= + 2023-08-15T01:39:57Z INFO plugin.opni.alerting extensions/embedded_server.go:88 starting opni embedded server {"addr": ":37795"} + ts=2023-08-15T01:39:57.782Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=36609 + ts=2023-08-15T01:39:57.782Z caller=delegate.go:236 level=debug component=cluster received=NotifyJoin node=01H7VDK16PBHAH6DXXGH2DBC9M addr=172.31.7.66:36609 + ts=2023-08-15T01:39:57.782Z caller=nflog.go:275 level=debug component=nflog msg="notification log snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/496233c9-b52c-41ec-873b-750058418da5/nflog: no such file or directory" + ts=2023-08-15T01:39:57.783Z caller=silence.go:353 level=debug component=silences msg="silences snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/496233c9-b52c-41ec-873b-750058418da5/silences: no such file or directory" + ts=2023-08-15T01:39:57.783Z caller=cluster.go:266 level=debug component=cluster msg="joined cluster" peers=0 + ts=2023-08-15T01:39:57.783Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:39:57.816Z caller=alertmanager_main.go:413 level=debug externalURL=http://ip-172-31-7-66.us-east-2.compute.internal:45495 + ts=2023-08-15T01:39:57.816Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/webhook-4e35a247-5d11-489e-83c8-40fa597e3e97/config-FFdQc2CufUZM7fS4V7Hxkj + ts=2023-08-15T01:39:57.817Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/webhook-4e35a247-5d11-489e-83c8-40fa597e3e97/config-FFdQc2CufUZM7fS4V7Hxkj + ts=2023-08-15T01:39:57.818Z caller=alertmanager_main.go:543 level=debug routePrefix=/ + ts=2023-08-15T01:39:57.819Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:45495 + ts=2023-08-15T01:39:57.819Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:45495 + 2023-08-15T01:39:58Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:39:58Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:39:58Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:39:58Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:39:58Z INFO test.env AlertManager started {"address": "http://localhost:45495", "opni-address": "http://localhost:824683462744"} + STEP: sending alerts to each condition in the router @ 08/15/23 01:39:58.684 + ts=2023-08-15T01:39:58.685Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[f5800cc][active] + ts=2023-08-15T01:39:58.686Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[843c13a][active] + ts=2023-08-15T01:39:58.686Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[b92943f][active] + STEP: getting the AlertManager state @ 08/15/23 01:39:58.687 + 2023-08-15T01:39:59Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:39:59Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:39:59Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:39:59Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + ts=2023-08-15T01:39:59.686Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"0492e20c-7f62-45e6-a1df-458561ea691e\", test=\"0492e20c-7f62-45e6-a1df-458561ea691e\"}" msg=flushing alerts=[[f5800cc][active]] + ts=2023-08-15T01:39:59.686Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:39:59.686Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:39:59.686Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"0492e20c-7f62-45e6-a1df-458561ea691e\"}:{opni_uuid=\"0492e20c-7f62-45e6-a1df-458561ea691e\", test=\"0492e20c-7f62-45e6-a1df-458561ea691e\"}" msg=flushing alerts=[[f5800cc][active]] + ts=2023-08-15T01:39:59.687Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + ts=2023-08-15T01:39:59.687Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + STEP: getting the AlertManager state @ 08/15/23 01:39:59.691 + ts=2023-08-15T01:39:59.784Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.001099747s + 2023-08-15T01:40:00Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:00Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:00Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:00Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:00.694 + 2023-08-15T01:40:01Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:01Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:01Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:01Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:01.698 + ts=2023-08-15T01:40:01.785Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=4.002260686s + 2023-08-15T01:40:02Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:02Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:02Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:02Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:02.702 + 2023-08-15T01:40:03Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:03Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:03Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:03Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:03.706 + ts=2023-08-15T01:40:03.786Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=6.003212044s + 2023-08-15T01:40:04Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:04Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:04Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:04Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:04.71 + 2023-08-15T01:40:05Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:05Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:05Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:05Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:05.714 + ts=2023-08-15T01:40:05.787Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=8.003730932s + 2023-08-15T01:40:06Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:06Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:06Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:06Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:06.718 + 2023-08-15T01:40:07Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:07Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:07Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:07Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:07.721 + ts=2023-08-15T01:40:07.788Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004699276s + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__0492e20c-7f62-45e6-a1df-458561ea691e integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__a135cbb4-f039-4f29-9153-dfe0ae62dd32 integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__22e67d2a-f6c0-461e-8a14-ee6be7004cfd integration=webhook[1] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__22e67d2a-f6c0-461e-8a14-ee6be7004cfd integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__a135cbb4-f039-4f29-9153-dfe0ae62dd32 integration=webhook[2] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.792Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"0492e20c-7f62-45e6-a1df-458561ea691e\"}:{opni_uuid=\"0492e20c-7f62-45e6-a1df-458561ea691e\", test=\"0492e20c-7f62-45e6-a1df-458561ea691e\"}" msg=flushing alerts=[[f5800cc][active]] + ts=2023-08-15T01:40:07.792Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:40:07.792Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.793Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__a135cbb4-f039-4f29-9153-dfe0ae62dd32 integration=webhook[1] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:07.793Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + 2023-08-15T01:40:08Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:08Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:08Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:08Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:08.725 + STEP: deleting a random server endpoint @ 08/15/23 01:40:08.728 + STEP: expecting to successfully build a config @ 08/15/23 01:40:08.729 + STEP: checking that the config passes the marshalling validation @ 08/15/23 01:40:08.729 + STEP: checking that the config passes the unmarshalling validation @ 08/15/23 01:40:08.732 + STEP: writing the config file to a temporary directory @ 08/15/23 01:40:08.738 + STEP: Verifying that the config can be loaded by alertmanager @ 08/15/23 01:40:08.738 + 2023-08-15T01:40:08Z INFO test.env Waiting for alertmanager to start... + ts=2023-08-15T01:40:08.836Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:40:08.837Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:40:08.837Z caller=cluster.go:173 level=debug component=cluster msg="resolved peers to following addresses" peers= + 2023-08-15T01:40:08Z INFO plugin.opni.alerting extensions/embedded_server.go:88 starting opni embedded server {"addr": ":45149"} + ts=2023-08-15T01:40:08.839Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=40461 + ts=2023-08-15T01:40:08.840Z caller=delegate.go:236 level=debug component=cluster received=NotifyJoin node=01H7VDKC07RJNFMDS3ZZAGXFFZ addr=172.31.7.66:40461 + ts=2023-08-15T01:40:08.840Z caller=nflog.go:275 level=debug component=nflog msg="notification log snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/a450ba5c-4085-4ee7-98c2-15fca48f4d02/nflog: no such file or directory" + ts=2023-08-15T01:40:08.840Z caller=silence.go:353 level=debug component=silences msg="silences snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/a450ba5c-4085-4ee7-98c2-15fca48f4d02/silences: no such file or directory" + ts=2023-08-15T01:40:08.840Z caller=cluster.go:266 level=debug component=cluster msg="joined cluster" peers=0 + ts=2023-08-15T01:40:08.840Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:40:08.872Z caller=alertmanager_main.go:413 level=debug externalURL=http://ip-172-31-7-66.us-east-2.compute.internal:43743 + ts=2023-08-15T01:40:08.873Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/webhook-4e35a247-5d11-489e-83c8-40fa597e3e97/config-qVtPRf6oS9VNSia69hynjA + ts=2023-08-15T01:40:08.873Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/webhook-4e35a247-5d11-489e-83c8-40fa597e3e97/config-qVtPRf6oS9VNSia69hynjA + ts=2023-08-15T01:40:08.875Z caller=alertmanager_main.go:543 level=debug routePrefix=/ + ts=2023-08-15T01:40:08.876Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:43743 + ts=2023-08-15T01:40:08.876Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:43743 + 2023-08-15T01:40:09Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:09Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:09Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:09Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:09Z INFO test.env AlertManager started {"address": "http://localhost:43743", "opni-address": "http://localhost:824688928536"} + STEP: sending alerts to each condition in the router @ 08/15/23 01:40:09.742 + ts=2023-08-15T01:40:09.743Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[f5800cc][active] + ts=2023-08-15T01:40:09.744Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[843c13a][active] + ts=2023-08-15T01:40:09.745Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[b92943f][active] + STEP: getting the AlertManager state @ 08/15/23 01:40:09.745 + 2023-08-15T01:40:10Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:10Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:10Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:10Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + ts=2023-08-15T01:40:10.744Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"0492e20c-7f62-45e6-a1df-458561ea691e\", test=\"0492e20c-7f62-45e6-a1df-458561ea691e\"}" msg=flushing alerts=[[f5800cc][active]] + ts=2023-08-15T01:40:10.745Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:40:10.745Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:40:10.745Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + ts=2023-08-15T01:40:10.745Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + STEP: getting the AlertManager state @ 08/15/23 01:40:10.748 + ts=2023-08-15T01:40:10.841Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000716812s + 2023-08-15T01:40:11Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:11Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:11Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:11Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:11.752 + 2023-08-15T01:40:12Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:12Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:12Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:12Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:12.756 + ts=2023-08-15T01:40:12.842Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=4.001435371s + 2023-08-15T01:40:13Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:13Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:13Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:13Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:13.759 + 2023-08-15T01:40:14Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:14Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:14Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:14Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:14.762 + ts=2023-08-15T01:40:14.843Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=6.002066983s + 2023-08-15T01:40:15Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:15Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:15Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:15Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:15.766 + 2023-08-15T01:40:16Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:16Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:16Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:16Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:16.77 + ts=2023-08-15T01:40:16.843Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=8.00277936s + 2023-08-15T01:40:17Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:17Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:17Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:17Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:17.774 + 2023-08-15T01:40:18Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:18Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:18Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:18Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:18.778 + ts=2023-08-15T01:40:18.844Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003654663s + ts=2023-08-15T01:40:18.846Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:18.847Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__a135cbb4-f039-4f29-9153-dfe0ae62dd32 integration=webhook[1] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:18.847Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:18.847Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__22e67d2a-f6c0-461e-8a14-ee6be7004cfd integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:18.848Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:40:18.848Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:18.848Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__a135cbb4-f039-4f29-9153-dfe0ae62dd32 integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:18.848Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + 2023-08-15T01:40:19Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:19Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:19Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:19Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:19.781 + STEP: updating an endpoint to another endpoint @ 08/15/23 01:40:19.784 + STEP: send an an alert to each specs @ 08/15/23 01:40:19.785 + STEP: expecting to successfully build a config @ 08/15/23 01:40:19.785 + STEP: checking that the config passes the marshalling validation @ 08/15/23 01:40:19.785 + STEP: checking that the config passes the unmarshalling validation @ 08/15/23 01:40:19.789 + STEP: writing the config file to a temporary directory @ 08/15/23 01:40:19.795 + STEP: Verifying that the config can be loaded by alertmanager @ 08/15/23 01:40:19.795 + 2023-08-15T01:40:19Z INFO test.env Waiting for alertmanager to start... + ts=2023-08-15T01:40:19.895Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:40:19.895Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + 2023-08-15T01:40:19Z INFO plugin.opni.alerting extensions/embedded_server.go:88 starting opni embedded server {"addr": ":45231"} + ts=2023-08-15T01:40:19.896Z caller=cluster.go:173 level=debug component=cluster msg="resolved peers to following addresses" peers= + ts=2023-08-15T01:40:19.898Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=37031 + ts=2023-08-15T01:40:19.899Z caller=delegate.go:236 level=debug component=cluster received=NotifyJoin node=01H7VDKPSTYQSVV8XAX0H1RNMS addr=172.31.7.66:37031 + ts=2023-08-15T01:40:19.899Z caller=nflog.go:275 level=debug component=nflog msg="notification log snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/66988909-cc6e-44b7-8113-9a61c7ec8ca5/nflog: no such file or directory" + ts=2023-08-15T01:40:19.899Z caller=silence.go:353 level=debug component=silences msg="silences snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/66988909-cc6e-44b7-8113-9a61c7ec8ca5/silences: no such file or directory" + ts=2023-08-15T01:40:19.899Z caller=cluster.go:266 level=debug component=cluster msg="joined cluster" peers=0 + ts=2023-08-15T01:40:19.899Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:40:19.929Z caller=alertmanager_main.go:413 level=debug externalURL=http://ip-172-31-7-66.us-east-2.compute.internal:35517 + ts=2023-08-15T01:40:19.929Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/webhook-4e35a247-5d11-489e-83c8-40fa597e3e97/config-9Z5Qi5BaPMc6fB5LtWxom3 + ts=2023-08-15T01:40:19.930Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/webhook-4e35a247-5d11-489e-83c8-40fa597e3e97/config-9Z5Qi5BaPMc6fB5LtWxom3 + ts=2023-08-15T01:40:19.932Z caller=alertmanager_main.go:543 level=debug routePrefix=/ + ts=2023-08-15T01:40:19.933Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:35517 + ts=2023-08-15T01:40:19.933Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:35517 + 2023-08-15T01:40:20Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:20Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:20Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:20Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:20Z INFO test.env AlertManager started {"address": "http://localhost:35517", "opni-address": "http://localhost:824693595272"} + STEP: sending alerts to each condition in the router @ 08/15/23 01:40:20.799 + ts=2023-08-15T01:40:20.801Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[f5800cc][active] + ts=2023-08-15T01:40:20.805Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[843c13a][active] + ts=2023-08-15T01:40:20.806Z caller=dispatch.go:163 level=debug component=dispatcher msg="Received alert" alert=[b92943f][active] + STEP: getting the AlertManager state @ 08/15/23 01:40:20.806 + 2023-08-15T01:40:21Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:21Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:21Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:21Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + ts=2023-08-15T01:40:21.802Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"0492e20c-7f62-45e6-a1df-458561ea691e\", test=\"0492e20c-7f62-45e6-a1df-458561ea691e\"}" msg=flushing alerts=[[f5800cc][active]] + ts=2023-08-15T01:40:21.806Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + ts=2023-08-15T01:40:21.806Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test!=\"\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:40:21.806Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:40:21.806Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + STEP: getting the AlertManager state @ 08/15/23 01:40:21.809 + ts=2023-08-15T01:40:21.900Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000906662s + 2023-08-15T01:40:22Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:22Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:22Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:22Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:22.812 + 2023-08-15T01:40:23Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:23Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:23Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:23Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:23.816 + ts=2023-08-15T01:40:23.900Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=4.001237221s + 2023-08-15T01:40:24Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:24Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:24Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:24Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:24.82 + 2023-08-15T01:40:25Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:25Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:25Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:25Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:25.823 + ts=2023-08-15T01:40:25.901Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=6.001417472s + 2023-08-15T01:40:26Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:26Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:26Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:26Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:26.827 + 2023-08-15T01:40:27Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:27Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:27Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:27Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:27.831 + ts=2023-08-15T01:40:27.901Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=8.002091736s + 2023-08-15T01:40:28Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:28Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:28Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:28Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:28.834 + 2023-08-15T01:40:29Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:29Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:29Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:29Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:29.838 + ts=2023-08-15T01:40:29.902Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002835787s + ts=2023-08-15T01:40:29.905Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__22e67d2a-f6c0-461e-8a14-ee6be7004cfd integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:29.905Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:29.905Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__a135cbb4-f039-4f29-9153-dfe0ae62dd32 integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:29.905Z caller=notify.go:751 level=debug component=dispatcher receiver=opni__test__a135cbb4-f039-4f29-9153-dfe0ae62dd32 integration=webhook[1] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:29.905Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}:{opni_uuid=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\", test=\"22e67d2a-f6c0-461e-8a14-ee6be7004cfd\"}" msg=flushing alerts=[[843c13a][active]] + ts=2023-08-15T01:40:29.905Z caller=dispatch.go:515 level=debug component=dispatcher aggrGroup="{}/{opni_datasource=\"\"}/{test!=\"\"}/{test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}:{opni_uuid=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\", test=\"a135cbb4-f039-4f29-9153-dfe0ae62dd32\"}" msg=flushing alerts=[[b92943f][active]] + ts=2023-08-15T01:40:29.906Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + ts=2023-08-15T01:40:29.906Z caller=notify.go:751 level=debug component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify success" attempts=1 + 2023-08-15T01:40:30Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:30Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:30Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:30Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + STEP: getting the AlertManager state @ 08/15/23 01:40:30.841 + STEP: dynamic-alert-routing step: expecting that the router can build the config @ 08/15/23 01:40:30.844 + STEP: dynamic-alert-routing step: expecting that the formed alertmanager config is correct @ 08/15/23 01:40:30.844 + STEP: checking that the config passes the marshalling validation @ 08/15/23 01:40:30.845 + STEP: checking that the config passes the unmarshalling validation @ 08/15/23 01:40:30.849 + STEP: writing the config file to a temporary directory @ 08/15/23 01:40:30.854 + STEP: Verifying that the config can be loaded by alertmanager @ 08/15/23 01:40:30.854 + 2023-08-15T01:40:30Z INFO test.env Waiting for alertmanager to start... + ts=2023-08-15T01:40:30.954Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:40:30.954Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:40:30.954Z caller=cluster.go:173 level=debug component=cluster msg="resolved peers to following addresses" peers= + 2023-08-15T01:40:30Z INFO plugin.opni.alerting extensions/embedded_server.go:88 starting opni embedded server {"addr": ":33703"} + ts=2023-08-15T01:40:30.957Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=42007 + ts=2023-08-15T01:40:30.957Z caller=delegate.go:236 level=debug component=cluster received=NotifyJoin node=01H7VDM1KDKHQSMB0F8575GNBZ addr=172.31.7.66:42007 + ts=2023-08-15T01:40:30.958Z caller=nflog.go:275 level=debug component=nflog msg="notification log snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/65d13bae-e61e-460f-854f-e03bfebcfaf1/nflog: no such file or directory" + ts=2023-08-15T01:40:30.958Z caller=silence.go:353 level=debug component=silences msg="silences snapshot file doesn't exist" err="open /tmp/opni-test-2132242568/alertmanager_data/65d13bae-e61e-460f-854f-e03bfebcfaf1/silences: no such file or directory" + ts=2023-08-15T01:40:30.958Z caller=cluster.go:266 level=debug component=cluster msg="joined cluster" peers=0 + ts=2023-08-15T01:40:30.958Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:40:30.989Z caller=alertmanager_main.go:413 level=debug externalURL=http://ip-172-31-7-66.us-east-2.compute.internal:36625 + ts=2023-08-15T01:40:30.989Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-d641c788-8419-4aa8-8c99-5c79ace97dc4/config-U9AJ2KpRYwn2padF6REDy4 + ts=2023-08-15T01:40:30.990Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-d641c788-8419-4aa8-8c99-5c79ace97dc4/config-U9AJ2KpRYwn2padF6REDy4 + ts=2023-08-15T01:40:30.992Z caller=alertmanager_main.go:543 level=debug routePrefix=/ + ts=2023-08-15T01:40:30.993Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:36625 + ts=2023-08-15T01:40:30.993Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:36625 + 2023-08-15T01:40:31Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:31Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:31Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:31Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:31Z INFO test.env AlertManager started {"address": "http://localhost:36625", "opni-address": "http://localhost:824705117080"} +• [34.192 seconds] +------------------------------ +metrics and alerting when When we use alerting on metrics should setup alertig & metrics clusters [integration] +/home/ec2-user/opni/test/plugins/alerting/metrics_test.go:37 + 2023-08-15T01:40:31Z INFO test.env Starting test environment + 2023-08-15T01:40:31Z INFO test.env Copying 7 files from embedded testdata/cortex to /tmp/opni-test-1343125945/cortex + 2023-08-15T01:40:31Z INFO test.env Waiting for jetstream to start... + [231568] 2023/08/15 01:40:31.870009 [INF] Starting nats-server + [231568] 2023/08/15 01:40:31.870058 [INF] Version: 2.9.21 + [231568] 2023/08/15 01:40:31.870062 [INF] Git: [b2e7725] + [231568] 2023/08/15 01:40:31.870066 [INF] Name: NBXVAIKYEU7FBTAO2673LMATBW2URPRH6FUVPMUF7F5QVLDRH27IZVNC + [231568] 2023/08/15 01:40:31.870072 [INF] Node: GU4SqiYQ + [231568] 2023/08/15 01:40:31.870076 [INF] ID: NBXVAIKYEU7FBTAO2673LMATBW2URPRH6FUVPMUF7F5QVLDRH27IZVNC + [231568] 2023/08/15 01:40:31.870098 [INF] Using configuration file: /tmp/opni-test-1343125945/jetstream/jetstream.conf + [231568] 2023/08/15 01:40:31.870791 [INF] Starting JetStream + [231568] 2023/08/15 01:40:31.870951 [INF] _ ___ _____ ___ _____ ___ ___ _ __ __ + [231568] 2023/08/15 01:40:31.871030 [INF] _ | | __|_ _/ __|_ _| _ \ __| /_\ | \/ | + [231568] 2023/08/15 01:40:31.871077 [INF] | || | _| | | \__ \ | | | / _| / _ \| |\/| | + [231568] 2023/08/15 01:40:31.871092 [INF] \__/|___| |_| |___/ |_| |_|_\___/_/ \_\_| |_| + [231568] 2023/08/15 01:40:31.871096 [INF] + [231568] 2023/08/15 01:40:31.871099 [INF] https://docs.nats.io/jetstream + [231568] 2023/08/15 01:40:31.871102 [INF] + [231568] 2023/08/15 01:40:31.871105 [INF] ---------------- JETSTREAM ---------------- + [231568] 2023/08/15 01:40:31.871112 [INF] Max Memory: 11.71 GB + [231568] 2023/08/15 01:40:31.871117 [INF] Max Storage: 5.29 GB + [231568] 2023/08/15 01:40:31.871120 [INF] Store Directory: "/tmp/opni-test-1343125945/jetstream/data/jetstream" + [231568] 2023/08/15 01:40:31.871123 [INF] ------------------------------------------- + [231568] 2023/08/15 01:40:31.871577 [INF] Listening for client connections on 0.0.0.0:32799 + [231568] 2023/08/15 01:40:31.871952 [INF] Server is ready + 2023-08-15T01:40:31Z INFO test.env Jetstream started + 2023-08-15T01:40:31Z DEBUG keyring loaded ephemeral key {"path": "/tmp/opni-test-1343125945/keyring/local-agent.json", "usage": "auth", "labels": {"opni.io/session-attribute":"local"}} + 2023-08-15T01:40:31Z DEBUG gateway found 0 plugins + 2023-08-15T01:40:31Z DEBUG gateway loaded plugin manifest {"plugins": 0} + 2023-08-15T01:40:31Z INFO gateway.cache compressing and archiving plugins... + 2023-08-15T01:40:31Z DEBUG gateway.cache added 0 new plugins to cache + 2023-08-15T01:40:31Z INFO gateway running plugin cache gc + 2023-08-15T01:40:31Z INFO gateway.update-server registering update handler for strategy "binary" + 2023-08-15T01:40:31Z INFO gateway.update-server registering update handler for strategy "kubernetes" + 2023-08-15T01:40:31Z INFO gateway.update-server registering update handler for strategy "noop" + 2023-08-15T01:40:31Z DEBUG gateway.grpc registering service {"service": "control.HealthListener"} + 2023-08-15T01:40:31Z DEBUG gateway.grpc registering internal service {"service": "stream.Delegate"} + 2023-08-15T01:40:31Z DEBUG gateway.grpc registering service {"service": "core.Pinger"} + 2023-08-15T01:40:31Z INFO test.env Loading gateway plugins... + 2023-08-15T01:40:31Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:31Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.backends.Capability", "opni.Metrics"]} + 2023-08-15T01:40:31Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.Metrics", "opni.backends.Capability", "opni.apiextensions.StreamAPIExtension", "opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.ManagementAPIExtension"]} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Metrics"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_metrics"} + 2023-08-15T01:40:31Z DEBUG plugin.metrics/pkg/gateway.stream internal gateway stream connected + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:31Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/alerting", "capability": "alerting"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_alerting"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Metrics"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:31Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/metrics", "capability": "metrics"} + 2023-08-15T01:40:31Z INFO mgmt got extension descriptor for service cortexadmin.CortexAdmin + 2023-08-15T01:40:31Z DEBUG gateway.grpc plugin rejected internal stream connection {"plugin": "plugin_alerting"} + 2023-08-15T01:40:31Z DEBUG plugin.metrics/pkg/gateway.stream calling client handler + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.System"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.System"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/trace", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/profile", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/block", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/heap", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/ready", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/healthy", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:40:31Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:31Z INFO mgmt loading service {"name": "cortexadmin.CortexAdmin"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/AllUserStats"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/WriteMetrics"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/Query"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/QueryRange"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetRule"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricMetadata"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ListRules"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/LoadRules"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/DeleteRule"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/FlushBlocks"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetSeriesMetrics"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricLabelSets"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexStatus"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexConfig"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ExtractRawSeries"} + 2023-08-15T01:40:31Z INFO mgmt loading http rules {"name": "cortexadmin.CortexAdmin", "rules": 17} + 2023-08-15T01:40:31Z DEBUG mgmt rule descriptors {"name": "cortexadmin.CortexAdmin", "rules": [{"http":{"Pattern":{"Get":"/all_user_stats"}},"method":{"name":"AllUserStats","input_type":".google.protobuf.Empty","output_type":".cortexadmin.UserIDStatsList","options":{}}},{"http":{"Pattern":{"Post":"/write_metrics"},"body":"*"},"method":{"name":"WriteMetrics","input_type":".cortexadmin.WriteRequest","output_type":".cortexadmin.WriteResponse","options":{}}},{"http":{"Pattern":{"Get":"/query"},"additional_bindings":[{"Pattern":{"Post":"/query"},"body":"*"}]},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query"},"body":"*"},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/query_range"},"additional_bindings":[{"Pattern":{"Post":"/query_range"},"body":"*"}]},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query_range"},"body":"*"},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/rules/{namespace}/{groupName}"}},"method":{"name":"GetRule","input_type":".cortexadmin.GetRuleRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/metadata"}},"method":{"name":"GetMetricMetadata","input_type":".cortexadmin.MetricMetadataRequest","output_type":".cortexadmin.MetricMetadata","options":{}}},{"http":{"Pattern":{"Get":"/rules"}},"method":{"name":"ListRules","input_type":".cortexadmin.ListRulesRequest","output_type":".cortexadmin.ListRulesResponse","options":{}}},{"http":{"Pattern":{"Post":"/rules"}},"method":{"name":"LoadRules","input_type":".cortexadmin.LoadRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/rules/{groupName}"}},"method":{"name":"DeleteRule","input_type":".cortexadmin.DeleteRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/flush_blocks"}},"method":{"name":"FlushBlocks","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/series/metadata"}},"method":{"name":"GetSeriesMetrics","input_type":".cortexadmin.SeriesRequest","output_type":".cortexadmin.SeriesInfoList","options":{}}},{"http":{"Pattern":{"Get":"/series/labels"}},"method":{"name":"GetMetricLabelSets","input_type":".cortexadmin.LabelRequest","output_type":".cortexadmin.MetricLabels","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetCortexStatus","input_type":".google.protobuf.Empty","output_type":".cortexadmin.CortexStatus","options":{}}},{"http":{"Pattern":{"Get":"/config"}},"method":{"name":"GetCortexConfig","input_type":".cortexadmin.ConfigRequest","output_type":".cortexadmin.ConfigResponse","options":{}}},{"http":{"Pattern":{"Get":"/series/raw"}},"method":{"name":"ExtractRawSeries","input_type":".cortexadmin.MatcherRequest","output_type":".cortexadmin.QueryResponse","options":{}}}]} + 2023-08-15T01:40:31Z INFO mgmt got extension descriptor for service cortexops.CortexOps + 2023-08-15T01:40:31Z INFO mgmt got extension descriptor for service alerting.AlertConditions + 2023-08-15T01:40:31Z INFO mgmt loading service {"name": "cortexops.CortexOps"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterConfiguration"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexops.CortexOps/ConfigureCluster"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterStatus"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/cortexops.CortexOps/UninstallCluster"} + 2023-08-15T01:40:31Z INFO mgmt loading http rules {"name": "cortexops.CortexOps", "rules": 4} + 2023-08-15T01:40:31Z DEBUG mgmt rule descriptors {"name": "cortexops.CortexOps", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".cortexops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".cortexops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".cortexops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:40:31Z INFO mgmt got extension descriptor for service remoteread.RemoteReadGateway + 2023-08-15T01:40:31Z INFO mgmt loading service {"name": "remoteread.RemoteReadGateway"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/AddTarget"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/EditTarget"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/RemoveTarget"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/ListTargets"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Start"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Stop"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/GetTargetStatus"} + 2023-08-15T01:40:31Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Discover"} + 2023-08-15T01:40:31Z INFO mgmt loading http rules {"name": "remoteread.RemoteReadGateway", "rules": 8} + 2023-08-15T01:40:31Z DEBUG mgmt rule descriptors {"name": "remoteread.RemoteReadGateway", "rules": [{"http":{"Pattern":{"Post":"/targets"},"body":"*"},"method":{"name":"AddTarget","input_type":".remoteread.TargetAddRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Put":"/targets"},"body":"*"},"method":{"name":"EditTarget","input_type":".remoteread.TargetEditRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/targets/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"RemoveTarget","input_type":".remoteread.TargetRemoveRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/targets"}},"method":{"name":"ListTargets","input_type":".remoteread.TargetListRequest","output_type":".remoteread.TargetList","options":{}}},{"http":{"Pattern":{"Post":"/targets/start/{target.meta.clusterId}/{target.meta.name}"},"body":"*"},"method":{"name":"Start","input_type":".remoteread.StartReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/targets/stop/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"Stop","input_type":".remoteread.StopReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status"},"body":"*"},"method":{"name":"GetTargetStatus","input_type":".remoteread.TargetStatusRequest","output_type":".remoteread.TargetStatus","options":{}}},{"http":{"Pattern":{"Post":"/edit"},"body":"*"},"method":{"name":"Discover","input_type":".remoteread.DiscoveryRequest","output_type":".remoteread.DiscoveryResponse","options":{}}}]} + 2023-08-15T01:40:31Z INFO mgmt got extension descriptor for service node.metrics.NodeConfiguration + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "node.metrics.NodeConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading http rules {"name": "node.metrics.NodeConfiguration", "rules": 4} + 2023-08-15T01:40:32Z DEBUG mgmt rule descriptors {"name": "node.metrics.NodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".node.metrics.MetricsCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".node.metrics.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:40:32Z INFO plugin.alerting using cluster driver {"driver": "test-environment"} + 2023-08-15T01:40:32Z INFO plugin.opni.alerting starting opni embedded server {"addr": "127.0.0.1:44103"} + 2023-08-15T01:40:32Z INFO plugin.metrics.cortex-http configuring http api server + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /ready", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "GET /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PUT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "PATCH /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "HEAD /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "DELETE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "DELETE /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z DEBUG gateway.http configured route for plugin {"route": "TRACE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "alerting.AlertConditions"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionGroups"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CreateAlertCondition"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/GetAlertCondition"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditions"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/UpdateAlertCondition"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionChoices"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeleteAlertCondition"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/AlertConditionStatus"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionsWithStatus"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CloneTo"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ActivateSilence"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeactivateSilence"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertConditions/Timeline"} + 2023-08-15T01:40:32Z INFO mgmt loading http rules {"name": "alerting.AlertConditions", "rules": 13} + 2023-08-15T01:40:32Z DEBUG mgmt rule descriptors {"name": "alerting.AlertConditions", "rules": [{"http":{"Pattern":{"Get":"/groups"}},"method":{"name":"ListAlertConditionGroups","input_type":".google.protobuf.Empty","output_type":".core.ReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertCondition","input_type":".alerting.AlertCondition","output_type":".alerting.ConditionReference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertCondition","input_type":".alerting.ConditionReference","output_type":".alerting.AlertCondition","options":{}}},{"http":{"Pattern":{"Post":"/list"},"body":"*"},"method":{"name":"ListAlertConditions","input_type":".alerting.ListAlertConditionRequest","output_type":".alerting.AlertConditionList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertCondition","input_type":".alerting.UpdateAlertConditionRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/choices"}},"method":{"name":"ListAlertConditionChoices","input_type":".alerting.AlertDetailChoicesRequest","output_type":".alerting.ListAlertTypeDetails","options":{}}},{"http":{"Pattern":{"Delete":"/configure"}},"method":{"name":"DeleteAlertCondition","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status/{id}"}},"method":{"name":"AlertConditionStatus","input_type":".alerting.ConditionReference","output_type":".alerting.AlertStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/list/withStatus"},"body":"*"},"method":{"name":"ListAlertConditionsWithStatus","input_type":".alerting.ListStatusRequest","output_type":".alerting.ListStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/clone"},"body":"*"},"method":{"name":"CloneTo","input_type":".alerting.CloneToRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/silences"},"body":"*"},"method":{"name":"ActivateSilence","input_type":".alerting.SilenceRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/silences"}},"method":{"name":"DeactivateSilence","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/timeline"}},"method":{"name":"Timeline","input_type":".alerting.TimelineRequest","output_type":".alerting.TimelineResponse","options":{}}}]} + 2023-08-15T01:40:32Z INFO mgmt got extension descriptor for service alerting.AlertEndpoints + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "alerting.AlertEndpoints"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/CreateAlertEndpoint"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/GetAlertEndpoint"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ToggleNotifications"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ListAlertEndpoints"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/UpdateAlertEndpoint"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/DeleteAlertEndpoint"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/TestAlertEndpoint"} + 2023-08-15T01:40:32Z INFO mgmt loading http rules {"name": "alerting.AlertEndpoints", "rules": 7} + 2023-08-15T01:40:32Z DEBUG mgmt rule descriptors {"name": "alerting.AlertEndpoints", "rules": [{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertEndpoint","input_type":".alerting.AlertEndpoint","output_type":".core.Reference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertEndpoint","input_type":".core.Reference","output_type":".alerting.AlertEndpoint","options":{}}},{"http":{"Pattern":{"Post":"/toggleNotifications"},"body":"*"},"method":{"name":"ToggleNotifications","input_type":".alerting.ToggleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/list"}},"method":{"name":"ListAlertEndpoints","input_type":".alerting.ListAlertEndpointsRequest","output_type":".alerting.AlertEndpointList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertEndpoint","input_type":".alerting.UpdateAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/delete"},"body":"*"},"method":{"name":"DeleteAlertEndpoint","input_type":".alerting.DeleteAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/test"},"body":"*"},"method":{"name":"TestAlertEndpoint","input_type":".alerting.TestAlertEndpointRequest","output_type":".alerting.TestAlertEndpointResponse","options":{}}}]} + 2023-08-15T01:40:32Z INFO mgmt got extension descriptor for service alerting.AlertNotifications + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "alerting.AlertNotifications"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/TriggerAlerts"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ResolveAlerts"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/PushNotification"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListNotifications"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListAlarmMessages"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListRoutingRelationships"} + 2023-08-15T01:40:32Z INFO mgmt loading http rules {"name": "alerting.AlertNotifications", "rules": 4} + 2023-08-15T01:40:32Z DEBUG mgmt rule descriptors {"name": "alerting.AlertNotifications", "rules": [{"http":{"Pattern":{"Post":"/trigger"},"body":"*"},"method":{"name":"TriggerAlerts","input_type":".alerting.TriggerAlertsRequest","output_type":".alerting.TriggerAlertsResponse","options":{}}},{"http":{"Pattern":{"Post":"/push"},"body":"*"},"method":{"name":"PushNotification","input_type":".alerting.Notification","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/notifications/list"},"body":"*"},"method":{"name":"ListNotifications","input_type":".alerting.ListNotificationRequest","output_type":".alerting.ListMessageResponse","options":{}}},{"http":{"Pattern":{"Post":"/alarms/list"},"body":"*"},"method":{"name":"ListAlarmMessages","input_type":".alerting.ListAlarmMessageRequest","output_type":".alerting.ListMessageResponse","options":{}}}]} + 2023-08-15T01:40:32Z INFO mgmt got extension descriptor for service alerting.ops.AlertingAdmin + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "alerting.ops.AlertingAdmin"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/ConfigureCluster"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterStatus"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/InstallCluster"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/UninstallCluster"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/Info"} + 2023-08-15T01:40:32Z INFO mgmt loading http rules {"name": "alerting.ops.AlertingAdmin", "rules": 6} + 2023-08-15T01:40:32Z DEBUG mgmt rule descriptors {"name": "alerting.ops.AlertingAdmin", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".alerting.ops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".alerting.ops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/install"}},"method":{"name":"InstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".alerting.ops.UninstallRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/info"}},"method":{"name":"Info","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ComponentInfo","options":{}}}]} + 2023-08-15T01:40:32Z INFO mgmt got extension descriptor for service alerting.ops.ConfigReconciler + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.ops.ConfigReconciler/SyncConfig"} + 2023-08-15T01:40:32Z INFO mgmt service has no http rules {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:40:32Z INFO mgmt got extension descriptor for service alerting.node.AlertingNodeConfiguration + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "alerting.node.AlertingNodeConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:40:32Z INFO mgmt loading http rules {"name": "alerting.node.AlertingNodeConfiguration", "rules": 4} + 2023-08-15T01:40:32Z DEBUG mgmt rule descriptors {"name": "alerting.node.AlertingNodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".alerting.node.AlertingCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".alerting.node.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:40:32Z INFO mgmt got extension descriptor for service alerting.node.NodeAlertingCapability + 2023-08-15T01:40:32Z INFO mgmt loading service {"name": "alerting.node.NodeAlertingCapability"} + 2023-08-15T01:40:32Z INFO mgmt loading method {"name": "/alerting.node.NodeAlertingCapability/Sync"} + 2023-08-15T01:40:32Z INFO mgmt loading http rules {"name": "alerting.node.NodeAlertingCapability", "rules": 1} + 2023-08-15T01:40:32Z DEBUG mgmt rule descriptors {"name": "alerting.node.NodeAlertingCapability", "rules": [{"http":{"Pattern":{"Post":"/v1/node/sync"},"body":"*"},"method":{"name":"Sync","input_type":".alerting.node.AlertingCapabilityConfig","output_type":".alerting.node.SyncResponse","options":{}}}]} + 2023-08-15T01:40:32Z INFO test.env Waiting for gateway to start... + 2023-08-15T01:40:32Z INFO mgmt management HTTP server starting {"address": "localhost:45639"} + 2023-08-15T01:40:32Z INFO gateway.http gateway HTTP server starting {"api": "127.0.0.1:45615", "metrics": "127.0.0.1:36647"} + 2023-08-15T01:40:32Z INFO test.env loaded 12 plugins + 2023-08-15T01:40:32Z INFO gateway.grpc gateway gRPC server starting {"address": "127.0.0.1:36691"} + 2023-08-15T01:40:32Z INFO mgmt management gRPC server starting {"address": "127.0.0.1:46283"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/tokens"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/tokens/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/watch"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/clusters/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/certs"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}/health"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/health/watch"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/clusters/{cluster.id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/roles"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/roles"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/roles/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/rolebindings"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/rolebindings"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/subjectaccess"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/apiextensions"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/config"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/config"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/capabilities"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/capabilities/{name}/installer"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/install"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/uninstall"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/status"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/status"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/cancel"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/dashboard/settings"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/dashboard/settings"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/all_user_stats"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/write_metrics"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules/{namespace}/{groupName}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/metadata"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/rules"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/CortexAdmin/rules/{groupName}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/flush_blocks"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/metadata"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/labels"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/status"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/config"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/raw"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/configuration"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/configure"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/status"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/uninstall"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:40:32Z INFO test.env Gateway started + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/RemoteReadGateway/targets/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/start/{target.meta.clusterId}/{target.meta.name}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/stop/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/status"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/edit"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertConditions/groups"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/configure"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertConditions/configure"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/choices"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/configure"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/status/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/withStatus"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/clone"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/silences"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/silences"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/timeline"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/list/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/toggleNotifications"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertEndpoints/list"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/delete"} + 2023-08-15T01:40:32Z DEBUG plugin.alerting-test-cluster-driver Syncer start : alerting-server --syncer.alertmanager.config.file=/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml --syncer.listen.address=:38629 --syncer.alertmanager.address=http://127.0.0.1:44497 --syncer.gateway.join.address=:46283 syncer {"config-file": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/test"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/trigger"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/push"} + 2023-08-15T01:40:32Z INFO plugin.alerting-test-cluster-driver Starting AlertManager {"config-file": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml", "alertmanager-port": 44497, "opni-port": 40425} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/notifications/list"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/alarms/list"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/configuration"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/configure"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/status"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/install"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/uninstall"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/info"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config/{id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:40:32Z DEBUG mgmt configured http handler {"method": "POST", "path": "/NodeAlertingCapability/v1/node/sync"} + ts=2023-08-15T01:40:32.162Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:40:32.162Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:40:32.165Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=39079 + ts=2023-08-15T01:40:32.166Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:40:32.197Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml + ts=2023-08-15T01:40:32.199Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml + ts=2023-08-15T01:40:32.201Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:44497 + ts=2023-08-15T01:40:32.201Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:44497 + 2023-08-15T01:40:32Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:32Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:32Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:32Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:32Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:32Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:32Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:32Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + ts=2023-08-15T01:40:32.959Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000539108s + 2023-08-15T01:40:33Z INFO plugin.alerting-test-cluster-driver Starting AlertManager Syncer {"config-file": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml", "port": 38629} + 2023-08-15T01:40:33Z INFO plugin.alerting-test-cluster-driver AlertManager started {"config-file": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml", "address": "http://127.0.0.1:44497"} + 2023-08-15T01:40:33Z INFO plugin.alerting updating known peers : [{http://127.0.0.1:44497 http://127.0.0.1:44103}] + 2023-08-15T01:40:33Z INFO plugin.metrics.metrics-backend node sync requested {"cluster": "(all)", "capability": "metrics"} + 2023-08-15T01:40:33Z INFO test.env Waiting for cortex to start... + 2023-08-15T01:40:33Z INFO test starting agent {"id": "agent1", "address": "127.0.0.1:46699", "version": "v2"} + 2023-08-15T01:40:33Z DEBUG agent using log level: debug + 2023-08-15T01:40:33Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:40:33Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent1 + 2023-08-15T01:40:33Z INFO agent bootstrap completed successfully + 2023-08-15T01:40:33Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:33Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:40:33Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:40:33Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:40:33Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:40:33Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension", "opni.backends.Capability"]} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:40:33Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:40:33Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:33Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:33Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:40:33Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:40:33Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:40:33Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:33Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:33Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:40:33Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:40:33Z INFO agent using unmanaged plugin loader + 2023-08-15T01:40:33Z INFO agent agent http server starting {"address": "127.0.0.1:46699"} + 2023-08-15T01:40:33Z INFO agent connecting to gateway... + 2023-08-15T01:40:33Z DEBUG gateway ratelimit: 50.000000 available + 2023-08-15T01:40:33Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:40:33Z DEBUG gateway-client authenticated {"id": "agent1"} + 2023-08-15T01:40:33Z DEBUG commands/alerting.go:72 syncer gateway join address:46283 + 2023-08-15T01:40:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:90 starting alerting syncer server as identity 6af4aa22-8ad6-4656-b187-103ece63a491 + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent1"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream connected + ts=2023-08-15T01:40:33.18017799Z caller=experimental.go:19 level=warn msg="experimental feature in use" feature=tenant-federation + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:108 acquired gateway alerting client + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:144 starting alerting syncer stream subscription + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:150 starting new stream subscription + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent1"} + 2023-08-15T01:40:33Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "6e33b99a-7400-45fd-8d69-2d0f7dddaf82"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:40:33Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "bd425fda-bda6-4fee-ad58-e7b42139b468"} + ts=2023-08-15T01:40:33.188315242Z caller=modules.go:331 level=warn msg="Worker address is empty in single binary mode. Attempting automatic worker configuration. If queries are unresponsive consider configuring the worker explicitly." address=127.0.0.1:37009 + 2023-08-15T01:40:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml"} + ts=2023-08-15T01:40:33.198Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml + ts=2023-08-15T01:40:33.199Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml"} + 2023-08-15T01:40:33Z INFO test.env Waiting for cortex to start... {"status": "503 Service Unavailable"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:40:33Z DEBUG gateway.sync agent connected {"id": "agent1"} + 2023-08-15T01:40:33Z INFO gateway.monitor received status update {"id": "agent1", "connected": true} + 2023-08-15T01:40:33Z DEBUG gateway.delegate agent connected {"id": "agent1"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:40:33Z INFO gateway.monitor received health update {"id": "agent1", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:40:33Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z INFO agent gateway connected + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:40:33Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:33Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent1", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:33Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:40:33Z INFO gateway.monitor received health update {"id": "agent1", "ready": true, "conditions": []} + 2023-08-15T01:40:33Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:33Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:33Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent1"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:40:33Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent1", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:33Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:33Z INFO test.env Cortex started {"httpAddress": "https://localhost:41355", "grpcAddress": "localhost:37009"} + 2023-08-15T01:40:33Z INFO test starting agent {"id": "agent2", "address": "127.0.0.1:40937", "version": "v2"} + 2023-08-15T01:40:33Z DEBUG agent using log level: debug + 2023-08-15T01:40:33Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:40:33Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent2 + 2023-08-15T01:40:33Z INFO agent bootstrap completed successfully + 2023-08-15T01:40:33Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:33Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:40:33Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:40:33Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:40:33Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:40:33Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:40:33Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:33Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:33Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:40:33Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:40:33Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:40:33Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:33Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:33Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:40:33Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:40:33Z INFO agent using unmanaged plugin loader + 2023-08-15T01:40:33Z INFO agent agent http server starting {"address": "127.0.0.1:40937"} + 2023-08-15T01:40:33Z INFO agent connecting to gateway... + 2023-08-15T01:40:33Z DEBUG gateway ratelimit: 50.000000 available + 2023-08-15T01:40:33Z DEBUG gateway-client authenticated {"id": "agent2"} + 2023-08-15T01:40:33Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:40:33Z INFO gateway.monitor received status update {"id": "agent2", "connected": true} + 2023-08-15T01:40:33Z DEBUG gateway.sync agent connected {"id": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:40:33Z DEBUG gateway.delegate agent connected {"id": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:40:33Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:33Z INFO agent gateway connected + 2023-08-15T01:40:33Z INFO gateway.monitor received health update {"id": "agent2", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:40:33Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent2", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:33Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:33Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:33Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent2", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "otel"} + 2023-08-15T01:40:33Z INFO gateway.monitor received health update {"id": "agent2", "ready": true, "conditions": []} + 2023-08-15T01:40:33Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:33Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:33Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:40:33Z INFO test starting agent {"id": "agent3", "address": "127.0.0.1:46631", "version": "v2"} + 2023-08-15T01:40:33Z DEBUG agent using log level: debug + 2023-08-15T01:40:33Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:40:33Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent3 + 2023-08-15T01:40:33Z INFO agent bootstrap completed successfully + 2023-08-15T01:40:33Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:33Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:40:33Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent3 is now active + 2023-08-15T01:40:33Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent3 is now active + 2023-08-15T01:40:33Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:40:33Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:40:33Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:33Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:40:33Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:33Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:33Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:40:33Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:40:33Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:40:33Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:33Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:33Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:40:33Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:40:33Z INFO agent using unmanaged plugin loader + 2023-08-15T01:40:33Z INFO agent agent http server starting {"address": "127.0.0.1:46631"} + 2023-08-15T01:40:33Z INFO agent connecting to gateway... + 2023-08-15T01:40:33Z DEBUG gateway ratelimit: 50.000000 available + 2023-08-15T01:40:33Z DEBUG gateway-client authenticated {"id": "agent3"} + 2023-08-15T01:40:33Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent3"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent3"} + 2023-08-15T01:40:33Z DEBUG gateway.sync agent connected {"id": "agent3"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:40:33Z INFO gateway.monitor received status update {"id": "agent3", "connected": true} + 2023-08-15T01:40:33Z DEBUG gateway.delegate agent connected {"id": "agent3"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:33Z INFO gateway.monitor received health update {"id": "agent3", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:40:33Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:40:33Z INFO agent gateway connected + 2023-08-15T01:40:33Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:33Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent3", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:33Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:33Z INFO gateway.monitor received health update {"id": "agent3", "ready": true, "conditions": []} + 2023-08-15T01:40:33Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent3"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:40:33Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent3", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent3", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent3", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent3", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:40:33Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:40:33Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent3", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:33Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent3"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent3"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:33Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:33Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:33Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} +• [1.834 seconds] +------------------------------ +metrics and alerting when When we use alerting on metrics should create prometheus query alerts that map to endpoints [integration] +/home/ec2-user/opni/test/plugins/alerting/metrics_test.go:87 + STEP: creating webhook endpoints for receiving the prometheus alerting @ 08/15/23 01:40:33.698 + STEP: creating prometheus query alert conditions @ 08/15/23 01:40:33.713 + STEP: making sure when metrics aren't installed, the conditions are invalid @ 08/15/23 01:40:33.732 +• [0.036 seconds] +------------------------------ +metrics and alerting when When we use alerting on metrics the metrics -> alerting pipeline should be functional [integration] +/home/ec2-user/opni/test/plugins/alerting/metrics_test.go:180 + STEP: installing the metrics capabilities @ 08/15/23 01:40:33.733 + 2023-08-15T01:40:33Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["metrics"]} + 2023-08-15T01:40:33Z DEBUG plugin.alerting ignoring sync request due to capability filter 'alerting' {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics received sync request + 2023-08-15T01:40:33Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:33Z INFO plugin.metrics.metrics-backend node sync requested {"cluster": "agent1", "capability": "metrics"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent1", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent2", "capabilities": ["metrics"]} + 2023-08-15T01:40:33Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:33Z DEBUG plugin.metrics received sync request + 2023-08-15T01:40:33Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:33Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent1"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics starting rule sync + 2023-08-15T01:40:33Z DEBUG plugin.metrics enabling http server + 2023-08-15T01:40:33Z DEBUG plugin.alerting ignoring sync request due to capability filter 'alerting' {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics configuring rule discovery + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule discovery configured + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule discovery notifier configured {"interval": "15m0s"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics starting rule group update notifier + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent1", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:33Z INFO test.env starting prometheus {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent2", "time": "2023-08-15T01:40:33Z"} + 2023-08-15T01:40:33Z INFO test.env Waiting for prometheus to start... + 2023-08-15T01:40:33Z INFO plugin.metrics.metrics-backend node sync requested {"cluster": "agent2", "capability": "metrics"} + 2023-08-15T01:40:33Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:33Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent3", "capabilities": ["metrics"]} + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent2", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:40:33Z INFO test.env starting prometheus {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics received sync request + 2023-08-15T01:40:33Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:33Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent2"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics starting rule sync + 2023-08-15T01:40:33Z DEBUG plugin.metrics enabling http server + 2023-08-15T01:40:33Z INFO test.env Waiting for prometheus to start... + 2023-08-15T01:40:33Z DEBUG plugin.metrics configuring rule discovery + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule discovery configured + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule discovery notifier configured {"interval": "15m0s"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics starting rule group update notifier + 2023-08-15T01:40:33Z DEBUG plugin.alerting ignoring sync request due to capability filter 'alerting' {"component": "health-cfg-sync"} + 2023-08-15T01:40:33Z INFO plugin.metrics.metrics-backend node sync requested {"cluster": "agent3", "capability": "metrics"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent3", "time": "2023-08-15T01:40:33Z"} + STEP: verifying the conditions move to the firing state @ 08/15/23 01:40:33.761 + STEP: making sure when metrics aren't installed, the conditions are invalid @ 08/15/23 01:40:33.761 + STEP: verifying the webhook endpoints have received the message @ 08/15/23 01:40:33.761 + 2023-08-15T01:40:33Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:33Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent3"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics starting rule sync + 2023-08-15T01:40:33Z DEBUG plugin.metrics enabling http server + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent3", "driver": "prometheus"} + 2023-08-15T01:40:33Z INFO test.env starting prometheus {"node": "agent3", "driver": "prometheus"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics configuring rule discovery + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule discovery configured + 2023-08-15T01:40:33Z DEBUG plugin.metrics rule discovery notifier configured {"interval": "15m0s"} + 2023-08-15T01:40:33Z DEBUG plugin.metrics starting rule group update notifier + 2023-08-15T01:40:33Z DEBUG test.env configuring node {"node": "agent3", "driver": "otel"} + 2023-08-15T01:40:33Z DEBUG test.env nothing to do (already stopped) {"node": "agent3", "driver": "otel"} + 2023-08-15T01:40:33Z INFO test.env Waiting for prometheus to start... + 2023-08-15T01:40:33Z INFO test.env Prometheus started {"address": "http://localhost:34255", "dir": "/tmp/opni-test-1343125945/prometheus/agent2", "agentId": "agent2"} + 2023-08-15T01:40:33Z INFO test.env started prometheus {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:40:33Z INFO test.env Prometheus started {"address": "http://localhost:43691", "dir": "/tmp/opni-test-1343125945/prometheus/agent1", "agentId": "agent1"} + 2023-08-15T01:40:33Z INFO test.env started prometheus {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:33Z INFO test.env Prometheus started {"address": "http://localhost:36529", "dir": "/tmp/opni-test-1343125945/prometheus/agent3", "agentId": "agent3"} + 2023-08-15T01:40:33Z INFO test.env started prometheus {"node": "agent3", "driver": "prometheus"} + 2023-08-15T01:40:33Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:33Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting received condition update: name:"agent-disconnect" description:"Alert when the downstream agent disconnects from the opni upstream" labels:"agent-disconnect" labels:"opni" labels:"_default" severity:Critical alertType:{system:{clusterId:{id:"agent3"} timeout:{seconds:1}}} lastUpdated:{seconds:1692063633 nanos:521667536} id:"qqYBdqk7WPkARoRmrZQ6vX" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onSystemConditionCreate": "qqYBdqk7WPkARoRmrZQ6vX"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting received condition update: name:"agent-disconnect" description:"Alert when the downstream agent disconnects from the opni upstream" labels:"agent-disconnect" labels:"opni" labels:"_default" severity:Critical alertType:{system:{clusterId:{id:"agent1"} timeout:{seconds:1}}} lastUpdated:{seconds:1692063633 nanos:126508511} id:"W8UA3Qt2is9jQ5jeTguLWP" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onSystemConditionCreate": "W8UA3Qt2is9jQ5jeTguLWP"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting Creating agent disconnect with timeout 1s {"component": "alarms", "onSystemConditionCreate": "qqYBdqk7WPkARoRmrZQ6vX"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting Creating agent disconnect with timeout 1s {"component": "alarms", "onSystemConditionCreate": "W8UA3Qt2is9jQ5jeTguLWP"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting name: uxELjJ6dzuXu3EbcnWZXgM-opni-alerting + rules: + - alert: uxELjJ6dzuXu3EbcnWZXgM + expr: opni:custom:prometheus_query{opni_uuid="uxELjJ6dzuXu3EbcnWZXgM"} + for: 10s + labels: + opni_promql: uxELjJ6dzuXu3EbcnWZXgM + opni_severity: Info + opni_uuid: uxELjJ6dzuXu3EbcnWZXgM + annotations: + OpniAlarmName: agent1 sanity metrics + OpniHeader: prometheus is receiving metrics + OpniSummary: see title + opni_clusterId: agent1 + opni_fingerprint: '{{ "ALERTS_FOR_STATE{opni_uuid=\"uxELjJ6dzuXu3EbcnWZXgM\"} OR vector(0)" | query | first | value | printf "%.0f" }}' + opni_goldenSignal: Custom + opni_uuid: uxELjJ6dzuXu3EbcnWZXgM + - record: opni:custom:prometheus_query + expr: sum(up > 0) > 0 + labels: + opni_uuid: uxELjJ6dzuXu3EbcnWZXgM + {"component": "alarms", "Expr": "user-query"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting received condition update: name:"agent-capability-unhealthy" description:"Alert when some downstream agent capability becomes unhealthy" labels:"agent-capability-health" labels:"opni" labels:"_default" severity:Critical alertType:{downstreamCapability:{clusterId:{id:"agent2"} capabilityState:"Failure" capabilityState:"Pending" for:{seconds:1}}} lastUpdated:{seconds:1692063633 nanos:323886561} id:"tsMswNN8UA3NDDHLuq4mdn" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onCapabilityStatusCreate": "tsMswNN8UA3NDDHLuq4mdn"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting Creating agent capability unhealthy with timeout 1s {"component": "alarms", "onCapabilityStatusCreate": "tsMswNN8UA3NDDHLuq4mdn"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting received condition update: name:"agent-capability-unhealthy" description:"Alert when some downstream agent capability becomes unhealthy" labels:"agent-capability-health" labels:"opni" labels:"_default" severity:Critical alertType:{downstreamCapability:{clusterId:{id:"agent3"} capabilityState:"Failure" capabilityState:"Pending" for:{seconds:1}}} lastUpdated:{seconds:1692063633 nanos:527070681} id:"kgWt69xi3LtA7U5u5WgCy7" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onCapabilityStatusCreate": "kgWt69xi3LtA7U5u5WgCy7"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting name: 6WkZhz8wcBYMCCrAh2mR5L-opni-alerting + rules: + - alert: 6WkZhz8wcBYMCCrAh2mR5L + expr: opni:custom:prometheus_query{opni_uuid="6WkZhz8wcBYMCCrAh2mR5L"} + for: 10s + labels: + opni_promql: 6WkZhz8wcBYMCCrAh2mR5L + opni_severity: Info + opni_uuid: 6WkZhz8wcBYMCCrAh2mR5L + annotations: + OpniAlarmName: agent2 sanity metrics + OpniHeader: prometheus is receiving metrics + OpniSummary: see title + opni_clusterId: agent2 + opni_fingerprint: '{{ "ALERTS_FOR_STATE{opni_uuid=\"6WkZhz8wcBYMCCrAh2mR5L\"} OR vector(0)" | query | first | value | printf "%.0f" }}' + opni_goldenSignal: Custom + opni_uuid: 6WkZhz8wcBYMCCrAh2mR5L + - record: opni:custom:prometheus_query + expr: sum(up > 0) > 0 + labels: + opni_uuid: 6WkZhz8wcBYMCCrAh2mR5L + {"component": "alarms", "Expr": "user-query"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting name: s8sq5YynSQuqNEMoqwL6mQ-opni-alerting + rules: + - alert: s8sq5YynSQuqNEMoqwL6mQ + expr: opni:custom:prometheus_query{opni_uuid="s8sq5YynSQuqNEMoqwL6mQ"} + for: 10s + labels: + opni_promql: s8sq5YynSQuqNEMoqwL6mQ + opni_severity: Info + opni_uuid: s8sq5YynSQuqNEMoqwL6mQ + annotations: + OpniAlarmName: agent3 sanity metrics + OpniHeader: prometheus is receiving metrics + OpniSummary: see title + opni_clusterId: agent3 + opni_fingerprint: '{{ "ALERTS_FOR_STATE{opni_uuid=\"s8sq5YynSQuqNEMoqwL6mQ\"} OR vector(0)" | query | first | value | printf "%.0f" }}' + opni_goldenSignal: Custom + opni_uuid: s8sq5YynSQuqNEMoqwL6mQ + - record: opni:custom:prometheus_query + expr: sum(up > 0) > 0 + labels: + opni_uuid: s8sq5YynSQuqNEMoqwL6mQ + {"component": "alarms", "Expr": "user-query"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting received condition update: name:"agent-disconnect" description:"Alert when the downstream agent disconnects from the opni upstream" labels:"agent-disconnect" labels:"opni" labels:"_default" severity:Critical alertType:{system:{clusterId:{id:"agent2"} timeout:{seconds:1}}} lastUpdated:{seconds:1692063633 nanos:319151968} id:"VHHsduREaDEvBj9kcPjSNJ" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onSystemConditionCreate": "VHHsduREaDEvBj9kcPjSNJ"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting received condition update: name:"agent-capability-unhealthy" description:"Alert when some downstream agent capability becomes unhealthy" labels:"agent-capability-health" labels:"opni" labels:"_default" severity:Critical alertType:{downstreamCapability:{clusterId:{id:"agent1"} capabilityState:"Failure" capabilityState:"Pending" for:{seconds:1}}} lastUpdated:{seconds:1692063633 nanos:139779253} id:"fETVFJdkn7WrPwHMiugGSN" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onCapabilityStatusCreate": "fETVFJdkn7WrPwHMiugGSN"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting Creating agent disconnect with timeout 1s {"component": "alarms", "onSystemConditionCreate": "VHHsduREaDEvBj9kcPjSNJ"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting Creating agent capability unhealthy with timeout 1s {"component": "alarms", "onCapabilityStatusCreate": "fETVFJdkn7WrPwHMiugGSN"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting Creating agent capability unhealthy with timeout 1s {"component": "alarms", "onCapabilityStatusCreate": "kgWt69xi3LtA7U5u5WgCy7"} + 2023-08-15T01:40:33Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:33Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:33Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml"} + ts=2023-08-15T01:40:33.955Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml + ts=2023-08-15T01:40:33.955Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml + 2023-08-15T01:40:33Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-1343125945/alertmanager-config-373e3580-58e9-4aa7-9189-0a23926d76d2/alertmanager.yaml"} + ts=2023-08-15T01:40:34.166Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000294759s + ts=2023-08-15T01:40:34.279461979Z caller=loader.go:117 level=warn msg="bucket index not found" user=agent2 + ts=2023-08-15T01:40:34.4027482Z caller=loader.go:117 level=warn msg="bucket index not found" user=agent1 + ts=2023-08-15T01:40:34.405248128Z caller=loader.go:117 level=warn msg="bucket index not found" user=agent3 + 2023-08-15T01:40:34Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:34Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:34Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:34Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:34Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:34Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + ts=2023-08-15T01:40:34.959Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=4.00108281s + 2023-08-15T01:40:34Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:216170439 {"component": "alarms", "onSystemConditionCreate": "W8UA3Qt2is9jQ5jeTguLWP"} + 2023-08-15T01:40:34Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:569546527 {"component": "alarms", "onSystemConditionCreate": "qqYBdqk7WPkARoRmrZQ6vX"} + 2023-08-15T01:40:34Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:371422007 {"component": "alarms", "onSystemConditionCreate": "VHHsduREaDEvBj9kcPjSNJ"} + 2023-08-15T01:40:34Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:216170439 {"component": "alarms", "onSystemConditionCreate": "W8UA3Qt2is9jQ5jeTguLWP"} + 2023-08-15T01:40:34Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:371422007 {"component": "alarms", "onSystemConditionCreate": "VHHsduREaDEvBj9kcPjSNJ"} + 2023-08-15T01:40:34Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:569546527 {"component": "alarms", "onSystemConditionCreate": "qqYBdqk7WPkARoRmrZQ6vX"} + 2023-08-15T01:40:35Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:371422007 {"component": "alarms", "onSystemConditionCreate": "VHHsduREaDEvBj9kcPjSNJ"} + 2023-08-15T01:40:35Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:216170439 {"component": "alarms", "onSystemConditionCreate": "W8UA3Qt2is9jQ5jeTguLWP"} + 2023-08-15T01:40:35Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063633 nanos:569546527 {"component": "alarms", "onSystemConditionCreate": "qqYBdqk7WPkARoRmrZQ6vX"} + 2023-08-15T01:40:35Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:35Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:35Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47382: EOF + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47248: read tcp 127.0.0.1:41355->127.0.0.1:47248: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47418: EOF + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47404: read tcp 127.0.0.1:41355->127.0.0.1:47404: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47332: read tcp 127.0.0.1:41355->127.0.0.1:47332: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47342: read tcp 127.0.0.1:41355->127.0.0.1:47342: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47352: read tcp 127.0.0.1:41355->127.0.0.1:47352: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47394: read tcp 127.0.0.1:41355->127.0.0.1:47394: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47420: read tcp 127.0.0.1:41355->127.0.0.1:47420: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47316: read tcp 127.0.0.1:41355->127.0.0.1:47316: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47368: read tcp 127.0.0.1:41355->127.0.0.1:47368: read: connection reset by peer + 2023/08/15 01:40:35 http: TLS handshake error from 127.0.0.1:47460: read tcp 127.0.0.1:41355->127.0.0.1:47460: read: connection reset by peer + 2023-08-15T01:40:35Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:35Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:35Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:35Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:35Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:35Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:35Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:35Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:36Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:36Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:36Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:36 http: TLS handshake error from 127.0.0.1:47536: read tcp 127.0.0.1:41355->127.0.0.1:47536: read: connection reset by peer + 2023/08/15 01:40:36 http: TLS handshake error from 127.0.0.1:47670: EOF + 2023/08/15 01:40:36 http: TLS handshake error from 127.0.0.1:47646: EOF + 2023-08-15T01:40:36Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:36Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:36Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:36Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:36Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:36Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:36Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:36Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:36Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:36Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:36Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + ts=2023-08-15T01:40:36.960Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=6.001822481s + 2023/08/15 01:40:36 http: TLS handshake error from 127.0.0.1:47738: EOF + 2023/08/15 01:40:36 http: TLS handshake error from 127.0.0.1:47764: read tcp 127.0.0.1:41355->127.0.0.1:47764: read: connection reset by peer + 2023/08/15 01:40:36 http: TLS handshake error from 127.0.0.1:47776: read tcp 127.0.0.1:41355->127.0.0.1:47776: read: connection reset by peer + 2023-08-15T01:40:37Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:37Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:37Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:37Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:37Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:37Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:37Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:37Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:37Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:37Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:37Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:37 http: TLS handshake error from 127.0.0.1:47862: read tcp 127.0.0.1:41355->127.0.0.1:47862: read: connection reset by peer + 2023/08/15 01:40:37 http: TLS handshake error from 127.0.0.1:47896: EOF + 2023/08/15 01:40:37 http: TLS handshake error from 127.0.0.1:47914: read tcp 127.0.0.1:41355->127.0.0.1:47914: read: connection reset by peer + 2023/08/15 01:40:37 http: TLS handshake error from 127.0.0.1:48016: read tcp 127.0.0.1:41355->127.0.0.1:48016: read: connection reset by peer + 2023-08-15T01:40:38Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:38Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:38Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:38Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:38Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:38Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:38Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:38Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + ts=2023-08-15T01:40:38.961Z caller=cluster.go:703 level=debug component=cluster msg="gossip looks settled" elapsed=8.002725938s + 2023/08/15 01:40:38 http: TLS handshake error from 127.0.0.1:48038: read tcp 127.0.0.1:41355->127.0.0.1:48038: read: connection reset by peer + 2023-08-15T01:40:38Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:38Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:38Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:38 http: TLS handshake error from 127.0.0.1:48100: EOF + 2023/08/15 01:40:38 http: TLS handshake error from 127.0.0.1:48114: EOF + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48228: EOF + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48218: read tcp 127.0.0.1:41355->127.0.0.1:48218: read: connection reset by peer + 2023-08-15T01:40:39Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:39Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:39Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:39Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:39Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:39Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:39Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:39Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:39Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:39Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:39Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48230: EOF + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48306: EOF + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48362: EOF + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48358: read tcp 127.0.0.1:41355->127.0.0.1:48358: read: connection reset by peer + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48374: EOF + 2023/08/15 01:40:39 http: TLS handshake error from 127.0.0.1:48346: EOF + 2023-08-15T01:40:40Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:40Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:40Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:40Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:40Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:40Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:40Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:40Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:40Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:40Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:40Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + ts=2023-08-15T01:40:40.962Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003741909s + 2023-08-15T01:40:40Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:40Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:41Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:41Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:41Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:41Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:41Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:41Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:41Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:41Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:41Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58026: EOF + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58090: read tcp 127.0.0.1:41355->127.0.0.1:58090: read: connection reset by peer + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58106: read tcp 127.0.0.1:41355->127.0.0.1:58106: read: connection reset by peer + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58154: read tcp 127.0.0.1:41355->127.0.0.1:58154: read: connection reset by peer + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58226: EOF + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58214: read tcp 127.0.0.1:41355->127.0.0.1:58214: read: connection reset by peer + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58110: read tcp 127.0.0.1:41355->127.0.0.1:58110: read: connection reset by peer + 2023/08/15 01:40:41 http: TLS handshake error from 127.0.0.1:58138: read tcp 127.0.0.1:41355->127.0.0.1:58138: read: connection reset by peer + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58134: EOF + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58188: read tcp 127.0.0.1:41355->127.0.0.1:58188: read: connection reset by peer + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58126: EOF + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58184: EOF + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58132: read tcp 127.0.0.1:41355->127.0.0.1:58132: read: connection reset by peer + 2023-08-15T01:40:42Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:42Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + ts=2023-08-15T01:40:42.168Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001481355s + 2023-08-15T01:40:42Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:42Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:42Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:42Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:42Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:42Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:42Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:42Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:42Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58228: read tcp 127.0.0.1:41355->127.0.0.1:58228: read: connection reset by peer + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58240: read tcp 127.0.0.1:41355->127.0.0.1:58240: read: connection reset by peer + 2023/08/15 01:40:42 http: TLS handshake error from 127.0.0.1:58266: EOF + 2023/08/15 01:40:43 http: TLS handshake error from 127.0.0.1:58318: EOF + 2023/08/15 01:40:43 http: TLS handshake error from 127.0.0.1:58354: EOF + 2023-08-15T01:40:43Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:43Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:43Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:43Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:43Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:43Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:43Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:41759 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46699", "scheme": "http"} + 2023-08-15T01:40:43Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:40117 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:40937", "scheme": "http"} + 2023-08-15T01:40:43Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:41759 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46699", "scheme": "http"} + 2023-08-15T01:40:43Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:40117 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:40937", "scheme": "http"} + 2023-08-15T01:40:43Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:42355 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46631", "scheme": "http"} + 2023-08-15T01:40:43Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:42355 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46631", "scheme": "http"} + 2023-08-15T01:40:43Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:43Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:43Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:43Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:43Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:43 http: TLS handshake error from 127.0.0.1:58412: EOF + 2023/08/15 01:40:43 http: TLS handshake error from 127.0.0.1:58406: EOF + 2023/08/15 01:40:43 http: TLS handshake error from 127.0.0.1:58426: EOF + 2023/08/15 01:40:43 http: TLS handshake error from 127.0.0.1:58458: read tcp 127.0.0.1:41355->127.0.0.1:58458: read: connection reset by peer + 2023/08/15 01:40:43 http: TLS handshake error from 127.0.0.1:58414: read tcp 127.0.0.1:41355->127.0.0.1:58414: read: connection reset by peer + 2023-08-15T01:40:44Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:44Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:44Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:44Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:44Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:44Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:44Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:44Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58524: read tcp 127.0.0.1:41355->127.0.0.1:58524: read: connection reset by peer + 2023-08-15T01:40:44Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:44Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:44Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58560: EOF + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58544: read tcp 127.0.0.1:41355->127.0.0.1:58544: read: connection reset by peer + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58574: read tcp 127.0.0.1:41355->127.0.0.1:58574: read: connection reset by peer + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58542: EOF + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58670: read tcp 127.0.0.1:41355->127.0.0.1:58670: read: connection reset by peer + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58602: read tcp 127.0.0.1:41355->127.0.0.1:58602: read: connection reset by peer + 2023/08/15 01:40:44 http: TLS handshake error from 127.0.0.1:58626: EOF + 2023-08-15T01:40:44Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:44Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:45Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:45Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:45Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:45Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:45Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:45Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023/08/15 01:40:45 http: TLS handshake error from 127.0.0.1:58694: EOF + 2023-08-15T01:40:45Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:45Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:45Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:40:46Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:46Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:46Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:46Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:46Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:46Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:46Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:46Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:46Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:46Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:46Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:46 http: TLS handshake error from 127.0.0.1:58950: EOF + 2023/08/15 01:40:46 http: TLS handshake error from 127.0.0.1:58936: read tcp 127.0.0.1:41355->127.0.0.1:58936: read: connection reset by peer + 2023/08/15 01:40:46 http: TLS handshake error from 127.0.0.1:59022: EOF + 2023/08/15 01:40:46 http: TLS handshake error from 127.0.0.1:59034: EOF + 2023/08/15 01:40:47 http: TLS handshake error from 127.0.0.1:59016: read tcp 127.0.0.1:41355->127.0.0.1:59016: read: connection reset by peer + 2023-08-15T01:40:47Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:47Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:47Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:47Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:47Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:47Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:47Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:47Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:47Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:47Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:47Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:47 http: TLS handshake error from 127.0.0.1:59076: EOF + 2023/08/15 01:40:47 http: TLS handshake error from 127.0.0.1:59090: EOF + 2023/08/15 01:40:47 http: TLS handshake error from 127.0.0.1:59140: read tcp 127.0.0.1:41355->127.0.0.1:59140: read: connection reset by peer + 2023/08/15 01:40:47 http: TLS handshake error from 127.0.0.1:59128: read tcp 127.0.0.1:41355->127.0.0.1:59128: read: connection reset by peer + 2023/08/15 01:40:48 http: TLS handshake error from 127.0.0.1:59260: EOF + 2023-08-15T01:40:48Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:48Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:48Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:48Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:48Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:48Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:48Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:48Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:40117 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:40937", "scheme": "http"} + 2023-08-15T01:40:48Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:48Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:41759 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46699", "scheme": "http"} + 2023-08-15T01:40:48Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:42355 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46631", "scheme": "http"} + 2023-08-15T01:40:48Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:48Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:48Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59372: read tcp 127.0.0.1:41355->127.0.0.1:59372: read: connection reset by peer + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59416: EOF + 2023-08-15T01:40:49Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:49Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:49Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:49Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:49Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:49Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:49Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:49Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:49Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:49Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:49Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59480: EOF + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59486: EOF + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59496: EOF + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59490: EOF + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59562: read tcp 127.0.0.1:41355->127.0.0.1:59562: read: connection reset by peer + 2023/08/15 01:40:49 http: TLS handshake error from 127.0.0.1:59548: EOF + 2023/08/15 01:40:50 http: TLS handshake error from 127.0.0.1:59632: read tcp 127.0.0.1:41355->127.0.0.1:59632: read: connection reset by peer + 2023/08/15 01:40:50 http: TLS handshake error from 127.0.0.1:59628: read tcp 127.0.0.1:41355->127.0.0.1:59628: read: connection reset by peer + 2023-08-15T01:40:50Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:50Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:50Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:50Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:50Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:50Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:50Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:50Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:50Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:50Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:50Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:50 http: TLS handshake error from 127.0.0.1:59756: read tcp 127.0.0.1:41355->127.0.0.1:59756: read: connection reset by peer + 2023/08/15 01:40:50 http: TLS handshake error from 127.0.0.1:59776: EOF + 2023/08/15 01:40:50 http: TLS handshake error from 127.0.0.1:59732: read tcp 127.0.0.1:41355->127.0.0.1:59732: read: connection reset by peer + 2023/08/15 01:40:50 http: TLS handshake error from 127.0.0.1:59716: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:59810: read tcp 127.0.0.1:41355->127.0.0.1:59810: read: connection reset by peer + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:59874: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:59842: EOF + 2023-08-15T01:40:51Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:51Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:51Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:51Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:51Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:51Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:51Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:51Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:51Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50392: EOF + 2023-08-15T01:40:51Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:51Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50380: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50374: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50408: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50540: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50504: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50508: EOF + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50422: read tcp 127.0.0.1:41355->127.0.0.1:50422: read: connection reset by peer + 2023/08/15 01:40:51 http: TLS handshake error from 127.0.0.1:50452: read tcp 127.0.0.1:41355->127.0.0.1:50452: read: connection reset by peer + 2023/08/15 01:40:52 http: TLS handshake error from 127.0.0.1:50556: EOF + 2023-08-15T01:40:52Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:52Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:52Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:52Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:52Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:52Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:52Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:52Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:52Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:52Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:52Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:52 http: TLS handshake error from 127.0.0.1:50582: read tcp 127.0.0.1:41355->127.0.0.1:50582: read: connection reset by peer + 2023/08/15 01:40:52 http: TLS handshake error from 127.0.0.1:50604: read tcp 127.0.0.1:41355->127.0.0.1:50604: read: connection reset by peer + 2023/08/15 01:40:52 http: TLS handshake error from 127.0.0.1:50618: EOF + 2023/08/15 01:40:52 http: TLS handshake error from 127.0.0.1:50622: read tcp 127.0.0.1:41355->127.0.0.1:50622: read: connection reset by peer + 2023/08/15 01:40:52 http: TLS handshake error from 127.0.0.1:50650: EOF + 2023-08-15T01:40:53Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:53Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:53Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:53Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:53Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:53Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:53Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:41759 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46699", "scheme": "http"} + 2023-08-15T01:40:53Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:40117 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:40937", "scheme": "http"} + 2023-08-15T01:40:53Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:42355 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46631", "scheme": "http"} + 2023-08-15T01:40:53Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:53Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:53Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023/08/15 01:40:53 http: TLS handshake error from 127.0.0.1:50770: EOF + 2023-08-15T01:40:53Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:53Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:53 http: TLS handshake error from 127.0.0.1:50828: EOF + 2023/08/15 01:40:53 http: TLS handshake error from 127.0.0.1:50840: read tcp 127.0.0.1:41355->127.0.0.1:50840: read: connection reset by peer + 2023-08-15T01:40:54Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:54Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:54Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:40117 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:40937", "scheme": "http"} + 2023-08-15T01:40:54Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:41759 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46699", "scheme": "http"} + 2023-08-15T01:40:54Z DEBUG api => {"method": "POST", "path": "/api/agent/push", "to": "127.0.0.1:42355 (plugin_metrics)", "for": "127.0.0.1", "host": "127.0.0.1:46631", "scheme": "http"} + 2023-08-15T01:40:54Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:54Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:54Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:54Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:54Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:54Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:54Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:50960: EOF + 2023-08-15T01:40:54Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:54Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:50998: read tcp 127.0.0.1:41355->127.0.0.1:50998: read: connection reset by peer + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:50978: EOF + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51026: read tcp 127.0.0.1:41355->127.0.0.1:51026: read: connection reset by peer + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51020: EOF + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51078: EOF + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51076: read tcp 127.0.0.1:41355->127.0.0.1:51076: read: connection reset by peer + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51100: EOF + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51050: EOF + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51104: EOF + 2023/08/15 01:40:54 http: TLS handshake error from 127.0.0.1:51134: EOF + 2023-08-15T01:40:55Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:55Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:55Z DEBUG api => {"method": "POST", "path": "/plugin_alerting/alertmanager/api/v2/*any", "to": "127.0.0.1:41281 (plugin_alerting)", "for": "127.0.0.1", "host": "127.0.0.1:45615", "scheme": "https"} + 2023-08-15T01:40:55Z DEBUG api => {"method": "POST", "path": "/plugin_alerting/alertmanager/api/v2/*any", "to": "127.0.0.1:41281 (plugin_alerting)", "for": "127.0.0.1", "host": "127.0.0.1:45615", "scheme": "https"} + 2023-08-15T01:40:55Z DEBUG api => {"method": "POST", "path": "/plugin_alerting/alertmanager/api/v2/*any", "to": "127.0.0.1:41281 (plugin_alerting)", "for": "127.0.0.1", "host": "127.0.0.1:45615", "scheme": "https"} +2023/08/15 01:40:55 http: TLS handshake error from 127.0.0.1:45752: EOF + 2023-08-15T01:40:55Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:55Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:55Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:55Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:55Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:55Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:55Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:55Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:55Z ERROR plugin.metrics.cortex-admin failed to get cluster status {"error": "rpc error: code = Internal desc = Get \"https://localhost:41477/services\": dial tcp 127.0.0.1:41477: connect: connection refused"} + 2023-08-15T01:40:55Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:55Z INFO plugin.alerting syncing 9 conditions {"component": "alarms"} + 2023-08-15T01:40:55Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8"} + 2023-08-15T01:40:55Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (9b6a81496f41fe62a50e55aa6309ffde77430baa7423cce8347375351a45c9f8) config message + 2023-08-15T01:40:55Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:55 http: TLS handshake error from 127.0.0.1:51236: EOF + 2023/08/15 01:40:55 http: TLS handshake error from 127.0.0.1:51216: EOF + 2023/08/15 01:40:55 http: TLS handshake error from 127.0.0.1:51280: read tcp 127.0.0.1:41355->127.0.0.1:51280: read: connection reset by peer + 2023/08/15 01:40:55 http: TLS handshake error from 127.0.0.1:51306: read tcp 127.0.0.1:41355->127.0.0.1:51306: read: connection reset by peer + 2023/08/15 01:40:56 http: TLS handshake error from 127.0.0.1:51358: EOF + 2023/08/15 01:40:56 http: TLS handshake error from 127.0.0.1:51342: EOF + 2023/08/15 01:40:56 http: TLS handshake error from 127.0.0.1:51308: EOF + 2023-08-15T01:40:56Z INFO plugin.alerting successfully synced (9/9) conditions {"component": "alarms"} + 2023-08-15T01:40:56Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + ts=2023-08-15T01:40:56.300Z caller=notify.go:740 level=warn component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 127.0.0.1:3000: connect: connection refused" + ts=2023-08-15T01:40:56.413Z caller=notify.go:740 level=warn component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 127.0.0.1:3000: connect: connection refused" + ts=2023-08-15T01:40:56.415Z caller=notify.go:740 level=warn component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 127.0.0.1:3000: connect: connection refused" + 2023-08-15T01:40:56Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:56Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:56Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:56Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "W8UA3Qt2is9jQ5jeTguLWP"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onCapabilityStatusCreate": "tsMswNN8UA3NDDHLuq4mdn"} + 2023-08-15T01:40:56Z INFO plugin.alerting shutting down cluster driver update handler + 2023-08-15T01:40:56Z INFO plugin.alerting exiting main sync loop + 2023-08-15T01:40:56Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "6e33b99a-7400-45fd-8d69-2d0f7dddaf82"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onCapabilityStatusCreate": "kgWt69xi3LtA7U5u5WgCy7"} + 2023-08-15T01:40:56Z DEBUG plugin.alerting closing cortex cluster status watcher... {"watcher": "cortex-cluster-status"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "qqYBdqk7WPkARoRmrZQ6vX"} + 2023-08-15T01:40:56Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "bd425fda-bda6-4fee-ad58-e7b42139b468"} + 2023-08-15T01:40:56Z INFO gateway shutting down plugins + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "W8UA3Qt2is9jQ5jeTguLWP"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "VHHsduREaDEvBj9kcPjSNJ"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "qqYBdqk7WPkARoRmrZQ6vX"} + 2023-08-15T01:40:56Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:56Z WARN plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onCapabilityStatusCreate": "fETVFJdkn7WrPwHMiugGSN"} + 2023-08-15T01:40:56Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "VHHsduREaDEvBj9kcPjSNJ"} + 2023-08-15T01:40:56Z DEBUG plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z WARN plugin.alerting failed to receive cluster health status from grpc stream, retrying... + 2023-08-15T01:40:56Z WARN plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z DEBUG plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z INFO test.env Cortex stopping... + 2023-08-15T01:40:56Z WARN plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z DEBUG plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z ERROR plugin.alerting failed to receive cluster event : rpc error: code = Canceled desc = context canceled + 2023-08-15T01:40:56Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:40:56Z ERROR test agent "agent2" exited: context canceled + 2023-08-15T01:40:56Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:56Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:56Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:40:56Z INFO gateway all plugins shut down + 2023-08-15T01:40:56Z ERROR test agent "agent3" exited: context canceled + 2023-08-15T01:40:56Z WARN test.env management server exited with error {"error": "context canceled"} + 2023-08-15T01:40:56Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:40:56Z DEBUG gateway.delegate agent disconnected {"id": "agent1"} + 2023-08-15T01:40:56Z DEBUG gateway.sync agent disconnected {"id": "agent1"} + 2023-08-15T01:40:56Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z DEBUG gateway.delegate agent disconnected {"id": "agent3"} + 2023-08-15T01:40:56Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z ERROR test agent "agent1" exited: context canceled + 2023-08-15T01:40:56Z DEBUG gateway.sync agent disconnected {"id": "agent2"} + 2023-08-15T01:40:56Z WARN gateway http server exited with error {"error": "context canceled"} + 2023-08-15T01:40:56Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:40:56Z DEBUG gateway.sync agent disconnected {"id": "agent3"} + 2023-08-15T01:40:56Z DEBUG gateway.delegate agent disconnected {"id": "agent2"} + 2023-08-15T01:40:56Z WARN test.env gateway server exited with error {"error": "context canceled"} + ts=2023-08-15T01:40:56.793Z caller=alertmanager_main.go:601 level=info msg="Received SIGTERM, exiting gracefully..." + 2023-08-15T01:40:56Z ERROR plugin.alerting read tcp 127.0.0.1:49688->127.0.0.1:32799: read: connection reset by peer + 2023-08-15T01:40:56Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error + 2023-08-15T01:40:56Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error + 2023-08-15T01:40:56Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:40:56Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} +• [23.076 seconds] +------------------------------ +Invalidated and clean up suite test when we update an alarm of the disconnect type should successfully update the group of the alarm [integration] +/home/ec2-user/opni/test/plugins/alerting/delete_test.go:154 + 2023-08-15T01:40:56Z INFO test.env Starting test environment + 2023-08-15T01:40:56Z INFO test.env Copying 7 files from embedded testdata/cortex to /tmp/opni-test-522595029/cortex + 2023-08-15T01:40:56Z INFO test.env Waiting for jetstream to start... + 2023-08-15T01:40:56Z INFO test.env Cortex stopped + [231646] 2023/08/15 01:40:56.819620 [INF] Starting nats-server + [231646] 2023/08/15 01:40:56.819685 [INF] Version: 2.9.21 + [231646] 2023/08/15 01:40:56.819689 [INF] Git: [b2e7725] + [231646] 2023/08/15 01:40:56.819696 [INF] Name: NBEONMMZHXYZAX6W2XLOBAQX7R4DWEQ27VEUM257CW53JY5AKV6D3QNU + [231646] 2023/08/15 01:40:56.819701 [INF] Node: 86Wk6YxY + [231646] 2023/08/15 01:40:56.819704 [INF] ID: NBEONMMZHXYZAX6W2XLOBAQX7R4DWEQ27VEUM257CW53JY5AKV6D3QNU + [231646] 2023/08/15 01:40:56.819720 [INF] Using configuration file: /tmp/opni-test-522595029/jetstream/jetstream.conf + [231646] 2023/08/15 01:40:56.820496 [INF] Starting JetStream + [231646] 2023/08/15 01:40:56.820638 [INF] _ ___ _____ ___ _____ ___ ___ _ __ __ + [231646] 2023/08/15 01:40:56.820654 [INF] _ | | __|_ _/ __|_ _| _ \ __| /_\ | \/ | + [231646] 2023/08/15 01:40:56.820658 [INF] | || | _| | | \__ \ | | | / _| / _ \| |\/| | + [231646] 2023/08/15 01:40:56.820661 [INF] \__/|___| |_| |___/ |_| |_|_\___/_/ \_\_| |_| + [231646] 2023/08/15 01:40:56.820664 [INF] + [231646] 2023/08/15 01:40:56.820667 [INF] https://docs.nats.io/jetstream + [231646] 2023/08/15 01:40:56.820670 [INF] + [231646] 2023/08/15 01:40:56.820673 [INF] ---------------- JETSTREAM ---------------- + [231646] 2023/08/15 01:40:56.820679 [INF] Max Memory: 11.71 GB + [231646] 2023/08/15 01:40:56.820683 [INF] Max Storage: 5.29 GB + [231646] 2023/08/15 01:40:56.820686 [INF] Store Directory: "/tmp/opni-test-522595029/jetstream/data/jetstream" + [231646] 2023/08/15 01:40:56.820689 [INF] ------------------------------------------- + [231646] 2023/08/15 01:40:56.821056 [INF] Listening for client connections on 0.0.0.0:42403 + [231646] 2023/08/15 01:40:56.821388 [INF] Server is ready + 2023-08-15T01:40:56Z INFO test.env Jetstream started + 2023-08-15T01:40:56Z DEBUG keyring loaded ephemeral key {"path": "/tmp/opni-test-522595029/keyring/local-agent.json", "usage": "auth", "labels": {"opni.io/session-attribute":"local"}} + 2023-08-15T01:40:56Z DEBUG gateway found 0 plugins + 2023-08-15T01:40:56Z DEBUG gateway loaded plugin manifest {"plugins": 0} + 2023-08-15T01:40:56Z INFO gateway.cache compressing and archiving plugins... + 2023-08-15T01:40:56Z DEBUG gateway.cache added 0 new plugins to cache + 2023-08-15T01:40:56Z INFO gateway running plugin cache gc + 2023-08-15T01:40:56Z INFO gateway.update-server registering update handler for strategy "binary" + 2023-08-15T01:40:56Z INFO gateway.update-server registering update handler for strategy "kubernetes" + 2023-08-15T01:40:56Z INFO gateway.update-server registering update handler for strategy "noop" + 2023-08-15T01:40:56Z DEBUG gateway.grpc registering service {"service": "control.HealthListener"} + 2023-08-15T01:40:56Z DEBUG gateway.grpc registering internal service {"service": "stream.Delegate"} + 2023-08-15T01:40:56Z DEBUG gateway.grpc registering service {"service": "core.Pinger"} + 2023-08-15T01:40:56Z INFO test.env Loading gateway plugins... + 2023-08-15T01:40:56Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.backends.Capability", "opni.Metrics", "opni.System"]} + 2023-08-15T01:40:56Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.apiextensions.ManagementAPIExtension", "opni.Metrics", "opni.backends.Capability", "opni.apiextensions.StreamAPIExtension", "opni.System", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.System"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_alerting"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.System"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.grpc plugin rejected internal stream connection {"plugin": "plugin_alerting"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_metrics"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG plugin.metrics/pkg/gateway.stream internal gateway stream connected + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/trace", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/profile", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/block", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/heap", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/ready", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/healthy", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service cortexadmin.CortexAdmin + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service alerting.AlertConditions + 2023-08-15T01:40:56Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/metrics", "capability": "metrics"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Metrics"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Metrics"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "cortexadmin.CortexAdmin"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/AllUserStats"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/WriteMetrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/Query"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/QueryRange"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetRule"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricMetadata"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ListRules"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/LoadRules"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/DeleteRule"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/FlushBlocks"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetSeriesMetrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricLabelSets"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexStatus"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexConfig"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ExtractRawSeries"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "cortexadmin.CortexAdmin", "rules": 17} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "cortexadmin.CortexAdmin", "rules": [{"http":{"Pattern":{"Get":"/all_user_stats"}},"method":{"name":"AllUserStats","input_type":".google.protobuf.Empty","output_type":".cortexadmin.UserIDStatsList","options":{}}},{"http":{"Pattern":{"Post":"/write_metrics"},"body":"*"},"method":{"name":"WriteMetrics","input_type":".cortexadmin.WriteRequest","output_type":".cortexadmin.WriteResponse","options":{}}},{"http":{"Pattern":{"Get":"/query"},"additional_bindings":[{"Pattern":{"Post":"/query"},"body":"*"}]},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query"},"body":"*"},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/query_range"},"additional_bindings":[{"Pattern":{"Post":"/query_range"},"body":"*"}]},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query_range"},"body":"*"},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/rules/{namespace}/{groupName}"}},"method":{"name":"GetRule","input_type":".cortexadmin.GetRuleRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/metadata"}},"method":{"name":"GetMetricMetadata","input_type":".cortexadmin.MetricMetadataRequest","output_type":".cortexadmin.MetricMetadata","options":{}}},{"http":{"Pattern":{"Get":"/rules"}},"method":{"name":"ListRules","input_type":".cortexadmin.ListRulesRequest","output_type":".cortexadmin.ListRulesResponse","options":{}}},{"http":{"Pattern":{"Post":"/rules"}},"method":{"name":"LoadRules","input_type":".cortexadmin.LoadRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/rules/{groupName}"}},"method":{"name":"DeleteRule","input_type":".cortexadmin.DeleteRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/flush_blocks"}},"method":{"name":"FlushBlocks","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/series/metadata"}},"method":{"name":"GetSeriesMetrics","input_type":".cortexadmin.SeriesRequest","output_type":".cortexadmin.SeriesInfoList","options":{}}},{"http":{"Pattern":{"Get":"/series/labels"}},"method":{"name":"GetMetricLabelSets","input_type":".cortexadmin.LabelRequest","output_type":".cortexadmin.MetricLabels","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetCortexStatus","input_type":".google.protobuf.Empty","output_type":".cortexadmin.CortexStatus","options":{}}},{"http":{"Pattern":{"Get":"/config"}},"method":{"name":"GetCortexConfig","input_type":".cortexadmin.ConfigRequest","output_type":".cortexadmin.ConfigResponse","options":{}}},{"http":{"Pattern":{"Get":"/series/raw"}},"method":{"name":"ExtractRawSeries","input_type":".cortexadmin.MatcherRequest","output_type":".cortexadmin.QueryResponse","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service cortexops.CortexOps + 2023-08-15T01:40:56Z DEBUG plugin.metrics/pkg/gateway.stream calling client handler + 2023-08-15T01:40:56Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:40:56Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:56Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/alerting", "capability": "alerting"} + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "cortexops.CortexOps"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexops.CortexOps/ConfigureCluster"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterStatus"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/cortexops.CortexOps/UninstallCluster"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "cortexops.CortexOps", "rules": 4} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "cortexops.CortexOps", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".cortexops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".cortexops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".cortexops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service remoteread.RemoteReadGateway + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "alerting.AlertConditions"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionGroups"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CreateAlertCondition"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/GetAlertCondition"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditions"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/UpdateAlertCondition"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionChoices"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeleteAlertCondition"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/AlertConditionStatus"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionsWithStatus"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CloneTo"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ActivateSilence"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeactivateSilence"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertConditions/Timeline"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "alerting.AlertConditions", "rules": 13} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "alerting.AlertConditions", "rules": [{"http":{"Pattern":{"Get":"/groups"}},"method":{"name":"ListAlertConditionGroups","input_type":".google.protobuf.Empty","output_type":".core.ReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertCondition","input_type":".alerting.AlertCondition","output_type":".alerting.ConditionReference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertCondition","input_type":".alerting.ConditionReference","output_type":".alerting.AlertCondition","options":{}}},{"http":{"Pattern":{"Post":"/list"},"body":"*"},"method":{"name":"ListAlertConditions","input_type":".alerting.ListAlertConditionRequest","output_type":".alerting.AlertConditionList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertCondition","input_type":".alerting.UpdateAlertConditionRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/choices"}},"method":{"name":"ListAlertConditionChoices","input_type":".alerting.AlertDetailChoicesRequest","output_type":".alerting.ListAlertTypeDetails","options":{}}},{"http":{"Pattern":{"Delete":"/configure"}},"method":{"name":"DeleteAlertCondition","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status/{id}"}},"method":{"name":"AlertConditionStatus","input_type":".alerting.ConditionReference","output_type":".alerting.AlertStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/list/withStatus"},"body":"*"},"method":{"name":"ListAlertConditionsWithStatus","input_type":".alerting.ListStatusRequest","output_type":".alerting.ListStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/clone"},"body":"*"},"method":{"name":"CloneTo","input_type":".alerting.CloneToRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/silences"},"body":"*"},"method":{"name":"ActivateSilence","input_type":".alerting.SilenceRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/silences"}},"method":{"name":"DeactivateSilence","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/timeline"}},"method":{"name":"Timeline","input_type":".alerting.TimelineRequest","output_type":".alerting.TimelineResponse","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service alerting.AlertEndpoints + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "alerting.AlertEndpoints"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/CreateAlertEndpoint"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/GetAlertEndpoint"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ToggleNotifications"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ListAlertEndpoints"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/UpdateAlertEndpoint"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/DeleteAlertEndpoint"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/TestAlertEndpoint"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "alerting.AlertEndpoints", "rules": 7} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "alerting.AlertEndpoints", "rules": [{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertEndpoint","input_type":".alerting.AlertEndpoint","output_type":".core.Reference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertEndpoint","input_type":".core.Reference","output_type":".alerting.AlertEndpoint","options":{}}},{"http":{"Pattern":{"Post":"/toggleNotifications"},"body":"*"},"method":{"name":"ToggleNotifications","input_type":".alerting.ToggleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/list"}},"method":{"name":"ListAlertEndpoints","input_type":".alerting.ListAlertEndpointsRequest","output_type":".alerting.AlertEndpointList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertEndpoint","input_type":".alerting.UpdateAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/delete"},"body":"*"},"method":{"name":"DeleteAlertEndpoint","input_type":".alerting.DeleteAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/test"},"body":"*"},"method":{"name":"TestAlertEndpoint","input_type":".alerting.TestAlertEndpointRequest","output_type":".alerting.TestAlertEndpointResponse","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service alerting.AlertNotifications + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "alerting.AlertNotifications"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/TriggerAlerts"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ResolveAlerts"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/PushNotification"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListNotifications"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListAlarmMessages"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListRoutingRelationships"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "alerting.AlertNotifications", "rules": 4} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "alerting.AlertNotifications", "rules": [{"http":{"Pattern":{"Post":"/trigger"},"body":"*"},"method":{"name":"TriggerAlerts","input_type":".alerting.TriggerAlertsRequest","output_type":".alerting.TriggerAlertsResponse","options":{}}},{"http":{"Pattern":{"Post":"/push"},"body":"*"},"method":{"name":"PushNotification","input_type":".alerting.Notification","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/notifications/list"},"body":"*"},"method":{"name":"ListNotifications","input_type":".alerting.ListNotificationRequest","output_type":".alerting.ListMessageResponse","options":{}}},{"http":{"Pattern":{"Post":"/alarms/list"},"body":"*"},"method":{"name":"ListAlarmMessages","input_type":".alerting.ListAlarmMessageRequest","output_type":".alerting.ListMessageResponse","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service alerting.ops.AlertingAdmin + 2023-08-15T01:40:56Z INFO plugin.metrics.cortex-http configuring http api server + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "remoteread.RemoteReadGateway"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/AddTarget"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/EditTarget"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/RemoveTarget"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/ListTargets"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Start"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Stop"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/GetTargetStatus"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Discover"} + 2023-08-15T01:40:56Z INFO plugin.alerting using cluster driver {"driver": "test-environment"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "remoteread.RemoteReadGateway", "rules": 8} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "remoteread.RemoteReadGateway", "rules": [{"http":{"Pattern":{"Post":"/targets"},"body":"*"},"method":{"name":"AddTarget","input_type":".remoteread.TargetAddRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Put":"/targets"},"body":"*"},"method":{"name":"EditTarget","input_type":".remoteread.TargetEditRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/targets/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"RemoveTarget","input_type":".remoteread.TargetRemoveRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/targets"}},"method":{"name":"ListTargets","input_type":".remoteread.TargetListRequest","output_type":".remoteread.TargetList","options":{}}},{"http":{"Pattern":{"Post":"/targets/start/{target.meta.clusterId}/{target.meta.name}"},"body":"*"},"method":{"name":"Start","input_type":".remoteread.StartReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/targets/stop/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"Stop","input_type":".remoteread.StopReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status"},"body":"*"},"method":{"name":"GetTargetStatus","input_type":".remoteread.TargetStatusRequest","output_type":".remoteread.TargetStatus","options":{}}},{"http":{"Pattern":{"Post":"/edit"},"body":"*"},"method":{"name":"Discover","input_type":".remoteread.DiscoveryRequest","output_type":".remoteread.DiscoveryResponse","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service node.metrics.NodeConfiguration + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "alerting.ops.AlertingAdmin"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterConfiguration"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/ConfigureCluster"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterStatus"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /ready", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "GET /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO plugin.opni.alerting starting opni embedded server {"addr": "127.0.0.1:43845"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PUT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "PATCH /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "HEAD /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/InstallCluster"} + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "node.metrics.NodeConfiguration"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "DELETE /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/UninstallCluster"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "node.metrics.NodeConfiguration", "rules": 4} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/Info"} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "node.metrics.NodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".node.metrics.MetricsCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".node.metrics.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "alerting.ops.AlertingAdmin", "rules": 6} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "alerting.ops.AlertingAdmin", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".alerting.ops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".alerting.ops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/install"}},"method":{"name":"InstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".alerting.ops.UninstallRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/info"}},"method":{"name":"Info","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ComponentInfo","options":{}}}]} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service alerting.ops.ConfigReconciler + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.ops.ConfigReconciler/SyncConfig"} + 2023-08-15T01:40:56Z INFO mgmt service has no http rules {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z DEBUG gateway.http configured route for plugin {"route": "TRACE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service alerting.node.AlertingNodeConfiguration + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "alerting.node.AlertingNodeConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "alerting.node.AlertingNodeConfiguration", "rules": 4} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "alerting.node.AlertingNodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".alerting.node.AlertingCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".alerting.node.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:40:56Z INFO mgmt got extension descriptor for service alerting.node.NodeAlertingCapability + 2023-08-15T01:40:56Z INFO mgmt loading service {"name": "alerting.node.NodeAlertingCapability"} + 2023-08-15T01:40:56Z INFO mgmt loading method {"name": "/alerting.node.NodeAlertingCapability/Sync"} + 2023-08-15T01:40:56Z INFO mgmt loading http rules {"name": "alerting.node.NodeAlertingCapability", "rules": 1} + 2023-08-15T01:40:56Z DEBUG mgmt rule descriptors {"name": "alerting.node.NodeAlertingCapability", "rules": [{"http":{"Pattern":{"Post":"/v1/node/sync"},"body":"*"},"method":{"name":"Sync","input_type":".alerting.node.AlertingCapabilityConfig","output_type":".alerting.node.SyncResponse","options":{}}}]} + 2023-08-15T01:40:56Z INFO test.env Waiting for gateway to start... + 2023-08-15T01:40:56Z INFO mgmt management HTTP server starting {"address": "localhost:38617"} + 2023-08-15T01:40:56Z INFO test.env loaded 12 plugins + 2023-08-15T01:40:56Z INFO mgmt management gRPC server starting {"address": "127.0.0.1:35907"} + 2023-08-15T01:40:56Z INFO gateway.grpc gateway gRPC server starting {"address": "127.0.0.1:43731"} + 2023-08-15T01:40:56Z INFO gateway.http gateway HTTP server starting {"api": "127.0.0.1:42365", "metrics": "127.0.0.1:45663"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/tokens"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/tokens/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/watch"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/clusters/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/certs"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}/health"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/health/watch"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/clusters/{cluster.id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/roles"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/roles"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/roles/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/rolebindings"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/rolebindings"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/subjectaccess"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/apiextensions"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/config"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/config"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/capabilities"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/capabilities/{name}/installer"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/install"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/uninstall"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/status"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/status"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/cancel"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/dashboard/settings"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/dashboard/settings"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/all_user_stats"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/write_metrics"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules/{namespace}/{groupName}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/metadata"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/rules"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/CortexAdmin/rules/{groupName}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/flush_blocks"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/metadata"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/labels"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/status"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/config"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/raw"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/configuration"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/configure"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/status"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/uninstall"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertConditions/groups"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/configure"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list"} + 2023-08-15T01:40:56Z INFO test.env Gateway started + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertConditions/configure"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/choices"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/configure"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/status/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/withStatus"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/clone"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/silences"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/silences"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/timeline"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/list/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/toggleNotifications"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertEndpoints/list"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/delete"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/test"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/trigger"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/push"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/notifications/list"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/alarms/list"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/RemoteReadGateway/targets/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/start/{target.meta.clusterId}/{target.meta.name}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/stop/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/status"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/edit"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/configuration"} + 2023-08-15T01:40:56Z DEBUG plugin.alerting-test-cluster-driver Syncer start : alerting-server --syncer.alertmanager.config.file=/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml --syncer.listen.address=:41087 --syncer.alertmanager.address=http://127.0.0.1:36163 --syncer.gateway.join.address=:35907 syncer {"config-file": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/configure"} + 2023-08-15T01:40:56Z INFO plugin.alerting-test-cluster-driver Starting AlertManager {"config-file": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml", "alertmanager-port": 36163, "opni-port": 33761} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/status"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/install"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/uninstall"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/info"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config/{id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:40:56Z DEBUG mgmt configured http handler {"method": "POST", "path": "/NodeAlertingCapability/v1/node/sync"} + ts=2023-08-15T01:40:57.061Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:40:57.061Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:40:57.064Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=46765 + ts=2023-08-15T01:40:57.065Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:40:57.096Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml + ts=2023-08-15T01:40:57.096Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml + ts=2023-08-15T01:40:57.099Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:36163 + ts=2023-08-15T01:40:57.099Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:36163 + 2023-08-15T01:40:57Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:57Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:57Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:57Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:57Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:57Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:57Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:57Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:57Z INFO plugin.alerting-test-cluster-driver Starting AlertManager Syncer {"config-file": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml", "port": 41087} + 2023-08-15T01:40:57Z INFO plugin.alerting-test-cluster-driver AlertManager started {"config-file": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml", "address": "http://127.0.0.1:36163"} + 2023-08-15T01:40:57Z INFO plugin.alerting updating known peers : [{http://127.0.0.1:36163 http://127.0.0.1:43845}] + 2023-08-15T01:40:57Z INFO test.env Waiting for cortex to start... + 2023-08-15T01:40:57Z INFO plugin.metrics.metrics-backend node sync requested {"cluster": "(all)", "capability": "metrics"} + 2023-08-15T01:40:57Z INFO test starting agent {"id": "agent1", "address": "127.0.0.1:36993", "version": "v2"} + 2023-08-15T01:40:57Z DEBUG agent using log level: debug + 2023-08-15T01:40:57Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:40:58Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent1 + 2023-08-15T01:40:58Z INFO agent bootstrap completed successfully + 2023-08-15T01:40:58Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:58Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:40:58Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:40:58Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:40:58Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.backends.Capability"]} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:40:58Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:40:58Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:58Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:58Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:40:58Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:40:58Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:40:58Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:58Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:58Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:40:58Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:40:58Z INFO agent using unmanaged plugin loader + 2023-08-15T01:40:58Z INFO agent agent http server starting {"address": "127.0.0.1:36993"} + 2023-08-15T01:40:58Z INFO agent connecting to gateway... + 2023-08-15T01:40:58Z DEBUG gateway ratelimit: 50.000000 available + 2023-08-15T01:40:58Z DEBUG gateway-client authenticated {"id": "agent1"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:40:58Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent1"} + 2023-08-15T01:40:58Z DEBUG commands/alerting.go:72 syncer gateway join address:35907 + 2023-08-15T01:40:58Z INFO plugin.alerting-syncer syncer/syncer_server.go:90 starting alerting syncer server as identity d698ddbc-b7c2-4866-868b-f42f0bb9ece1 + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent1"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:108 acquired gateway alerting client + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:144 starting alerting syncer stream subscription + 2023-08-15T01:40:58Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "f381c452-12ec-4c96-b432-54636f604a96"} + 2023-08-15T01:40:58Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "b1d328c9-a063-4e6f-993d-10490e6e76fb"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:150 starting new stream subscription + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + ts=2023-08-15T01:40:58.098819998Z caller=experimental.go:19 level=warn msg="experimental feature in use" feature=tenant-federation + 2023-08-15T01:40:58Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml"} + ts=2023-08-15T01:40:58.106Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml + ts=2023-08-15T01:40:58.106Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml + ts=2023-08-15T01:40:58.108418462Z caller=modules.go:331 level=warn msg="Worker address is empty in single binary mode. Attempting automatic worker configuration. If queries are unresponsive consider configuring the worker explicitly." address=127.0.0.1:33573 + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:58Z DEBUG gateway.sync agent connected {"id": "agent1"} + 2023-08-15T01:40:58Z INFO gateway.monitor received status update {"id": "agent1", "connected": true} + 2023-08-15T01:40:58Z DEBUG gateway.delegate agent connected {"id": "agent1"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:58Z INFO agent gateway connected + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:40:58Z INFO gateway.monitor received health update {"id": "agent1", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:40:58Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:40:58Z"} + 2023-08-15T01:40:58Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:58Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z INFO test.env Cortex started {"httpAddress": "https://localhost:46751", "grpcAddress": "localhost:33573"} + 2023-08-15T01:40:58Z INFO test starting agent {"id": "agent2", "address": "127.0.0.1:46475", "version": "v2"} + 2023-08-15T01:40:58Z DEBUG agent using log level: debug + 2023-08-15T01:40:58Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:40:58Z INFO gateway.monitor received health update {"id": "agent1", "ready": true, "conditions": []} + 2023-08-15T01:40:58Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:40:58Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:40:58Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:58Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent1", "time": "2023-08-15T01:40:58Z"} + 2023-08-15T01:40:58Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:58Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:58Z DEBUG test.env configuring node {"node": "agent1", "driver": "otel"} + 2023-08-15T01:40:58Z DEBUG test.env configuring node {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:58Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "otel"} + 2023-08-15T01:40:58Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent1"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:40:58Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:58Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:58Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent2 + 2023-08-15T01:40:58Z INFO agent bootstrap completed successfully + 2023-08-15T01:40:58Z WARN jetstream disconnected from jetstream + 2023-08-15T01:40:58Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:40:58Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:40:58Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:40:58Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:40:58Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:40:58Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:40:58Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:40:58Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:40:58Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:58Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:58Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:40:58Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:40:58Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:40:58Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:40:58Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:40:58Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:40:58Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:40:58Z INFO agent using unmanaged plugin loader + 2023-08-15T01:40:58Z INFO agent agent http server starting {"address": "127.0.0.1:46475"} + 2023-08-15T01:40:58Z INFO agent connecting to gateway... + 2023-08-15T01:40:58Z DEBUG gateway ratelimit: 50.000000 available + 2023-08-15T01:40:58Z DEBUG gateway-client authenticated {"id": "agent2"} + 2023-08-15T01:40:58Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent2"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent2"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:40:58Z DEBUG gateway.sync agent connected {"id": "agent2"} + 2023-08-15T01:40:58Z INFO gateway.monitor received status update {"id": "agent2", "connected": true} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:58Z DEBUG gateway.delegate agent connected {"id": "agent2"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:40:58Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:58Z INFO agent gateway connected + 2023-08-15T01:40:58Z INFO gateway.monitor received health update {"id": "agent2", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:40:58Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent2", "time": "2023-08-15T01:40:58Z"} + 2023-08-15T01:40:58Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:58Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:58Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent2"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:40:58Z DEBUG test.env configuring node {"node": "agent2", "driver": "otel"} + 2023-08-15T01:40:58Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "otel"} + 2023-08-15T01:40:58Z DEBUG test.env configuring node {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:40:58Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:40:58Z INFO gateway.monitor received health update {"id": "agent2", "ready": true, "conditions": []} + 2023-08-15T01:40:58Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:58Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:40:58Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:40:58Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:40:58Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:40:58Z"} + 2023-08-15T01:40:58Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:40:58Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["metrics"]} + 2023-08-15T01:40:58Z DEBUG plugin.alerting ignoring sync request due to capability filter 'alerting' {"component": "health-cfg-sync"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics received sync request + 2023-08-15T01:40:58Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:40:58Z INFO plugin.metrics.metrics-backend node sync requested {"cluster": "agent1", "capability": "metrics"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent1", "time": "2023-08-15T01:40:58Z"} + 2023-08-15T01:40:58Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:40:58Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent1"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics starting rule sync + 2023-08-15T01:40:58Z DEBUG test.env configuring node {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics enabling http server + 2023-08-15T01:40:58Z INFO test.env starting prometheus {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics configuring rule discovery + 2023-08-15T01:40:58Z DEBUG plugin.metrics rule discovery configured + 2023-08-15T01:40:58Z DEBUG plugin.metrics rule discovery notifier configured {"interval": "15m0s"} + 2023-08-15T01:40:58Z DEBUG plugin.metrics starting rule group update notifier + 2023-08-15T01:40:58Z DEBUG test.env configuring node {"node": "agent1", "driver": "otel"} + 2023-08-15T01:40:58Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "otel"} + STEP: verifying the only the default group exists @ 08/15/23 01:40:58.484 + 2023-08-15T01:40:58Z INFO test.env Waiting for prometheus to start... + 2023-08-15T01:40:58Z DEBUG plugin.alerting Updating alert condition id:"bGpeDG5QAFzsEZB5R9j8kk" {"component": "alarms", "handler": "UpdateAlertCondition"} + STEP: veriying the condition in the old group storage is not found @ 08/15/23 01:40:58.504 + STEP: veryfing the condition has been move to the new group storage @ 08/15/23 01:40:58.504 +• [1.697 seconds] +------------------------------ +Invalidated and clean up suite test when we delete an alarm of the metrics type should successfully submit the alarm for deletion [integration] +/home/ec2-user/opni/test/plugins/alerting/delete_test.go:199 + STEP: verifying the metrics conditions eventually are active successfully @ 08/15/23 01:40:58.511 + 2023/08/15 01:40:58 http: TLS handshake error from 127.0.0.1:40808: EOF + 2023-08-15T01:40:58Z INFO test.env Prometheus started {"address": "http://localhost:44699", "dir": "/tmp/opni-test-522595029/prometheus/agent1", "agentId": "agent1"} + 2023-08-15T01:40:58Z INFO test.env started prometheus {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:40:58Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:58Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:58Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:58Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:58Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "6af872f48e9930297eb9b7408690ad0524d9893461a72a1bd61fde0594914d2d"} + 2023-08-15T01:40:58Z INFO plugin.alerting syncing 6 conditions {"component": "alarms"} + 2023-08-15T01:40:58Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (6af872f48e9930297eb9b7408690ad0524d9893461a72a1bd61fde0594914d2d) config message + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml"} + ts=2023-08-15T01:40:58.890Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml + ts=2023-08-15T01:40:58.891Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml + 2023-08-15T01:40:58Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-522595029/alertmanager-config-a748e770-be6e-4ad4-9b20-7d72ed8c784d/alertmanager.yaml"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting received condition update: name:"agent disconnect" description:"some disconnect to remove" alertType:{system:{clusterId:{id:"agent1"} timeout:{seconds:600}}} silence:{} lastUpdated:{seconds:1692063658 nanos:493414883} id:"bGpeDG5QAFzsEZB5R9j8kk" metadata:{key:"opni.io/alarm-inactive" value:"true"} groupId:"new-group" {"component": "alarms", "onSystemConditionCreate": "bGpeDG5QAFzsEZB5R9j8kk"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting Creating agent disconnect with timeout 10m0s {"component": "alarms", "onSystemConditionCreate": "bGpeDG5QAFzsEZB5R9j8kk"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting name: cf9Lm3PynGemtUxhLZ2Ezk-opni-alerting + rules: + - alert: cf9Lm3PynGemtUxhLZ2Ezk + expr: opni:custom:prometheus_query{opni_uuid="cf9Lm3PynGemtUxhLZ2Ezk"} + for: 1s + labels: + opni_promql: cf9Lm3PynGemtUxhLZ2Ezk + opni_severity: Info + opni_uuid: cf9Lm3PynGemtUxhLZ2Ezk + annotations: + OpniAlarmName: test + OpniHeader: test + OpniSummary: Prometheus query + opni_clusterId: agent1 + opni_fingerprint: '{{ "ALERTS_FOR_STATE{opni_uuid=\"cf9Lm3PynGemtUxhLZ2Ezk\"} OR vector(0)" | query | first | value | printf "%.0f" }}' + opni_goldenSignal: Custom + opni_uuid: cf9Lm3PynGemtUxhLZ2Ezk + - record: opni:custom:prometheus_query + expr: sum(up > 0) > 0 + labels: + opni_uuid: cf9Lm3PynGemtUxhLZ2Ezk + {"component": "alarms", "Expr": "user-query"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting received condition update: name:"agent-disconnect" description:"Alert when the downstream agent disconnects from the opni upstream" labels:"agent-disconnect" labels:"opni" labels:"_default" severity:Critical alertType:{system:{clusterId:{id:"agent1"} timeout:{seconds:1}}} lastUpdated:{seconds:1692063658 nanos:28040087} id:"dCfr8i4Q6EHPvxZwW9gLt6" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onSystemConditionCreate": "dCfr8i4Q6EHPvxZwW9gLt6"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting Creating agent disconnect with timeout 1s {"component": "alarms", "onSystemConditionCreate": "dCfr8i4Q6EHPvxZwW9gLt6"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting received condition update: name:"agent-disconnect" description:"Alert when the downstream agent disconnects from the opni upstream" labels:"agent-disconnect" labels:"opni" labels:"_default" severity:Critical alertType:{system:{clusterId:{id:"agent2"} timeout:{seconds:1}}} lastUpdated:{seconds:1692063658 nanos:297438417} id:"34e6zCJWGxLSxNSP97cftd" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting Creating agent disconnect with timeout 1s {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting received condition update: name:"agent-capability-unhealthy" description:"Alert when some downstream agent capability becomes unhealthy" labels:"agent-capability-health" labels:"opni" labels:"_default" severity:Critical alertType:{downstreamCapability:{clusterId:{id:"agent2"} capabilityState:"Failure" capabilityState:"Pending" for:{seconds:1}}} lastUpdated:{seconds:1692063658 nanos:302402760} id:"stdRzQww4KPXDy5r3dVAh6" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onCapabilityStatusCreate": "stdRzQww4KPXDy5r3dVAh6"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting received condition update: name:"agent-capability-unhealthy" description:"Alert when some downstream agent capability becomes unhealthy" labels:"agent-capability-health" labels:"opni" labels:"_default" severity:Critical alertType:{downstreamCapability:{clusterId:{id:"agent1"} capabilityState:"Failure" capabilityState:"Pending" for:{seconds:1}}} lastUpdated:{seconds:1692063658 nanos:54065227} id:"FsECjbp6G7xntHLX5o9DjR" metadata:{key:"opni.io/alarm-inactive" value:"true"} {"component": "alarms", "onCapabilityStatusCreate": "FsECjbp6G7xntHLX5o9DjR"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting Creating agent capability unhealthy with timeout 1s {"component": "alarms", "onCapabilityStatusCreate": "stdRzQww4KPXDy5r3dVAh6"} + 2023-08-15T01:40:58Z DEBUG plugin.alerting Creating agent capability unhealthy with timeout 1s {"component": "alarms", "onCapabilityStatusCreate": "FsECjbp6G7xntHLX5o9DjR"} + 2023-08-15T01:40:58Z INFO plugin.alerting successfully synced (6/6) conditions {"component": "alarms"} + 2023-08-15T01:40:58Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40826: EOF + ts=2023-08-15T01:40:59.066Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000373264s + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40830: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40840: read tcp 127.0.0.1:46751->127.0.0.1:40840: read: connection reset by peer + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40846: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40858: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40864: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40872: read tcp 127.0.0.1:46751->127.0.0.1:40872: read: connection reset by peer + STEP: deleting the metrics condition @ 08/15/23 01:40:59.139 + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40878: EOF +• [0.645 seconds] +------------------------------ +Invalidated and clean up suite test when we delete an alarm of the metrics type should clean up the dependencies of this alarm [integration] +/home/ec2-user/opni/test/plugins/alerting/delete_test.go:258 + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40888: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40904: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40916: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40930: EOF + 2023-08-15T01:40:59Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:59Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:40:59Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:40:59Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + ts=2023-08-15T01:40:59.721968567Z caller=loader.go:117 level=warn msg="bucket index not found" user=agent1 + 2023-08-15T01:40:59Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:40:59Z INFO plugin.alerting syncing 6 conditions {"component": "alarms"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "6af872f48e9930297eb9b7408690ad0524d9893461a72a1bd61fde0594914d2d"} + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40938: read tcp 127.0.0.1:46751->127.0.0.1:40938: read: connection reset by peer + 2023-08-15T01:40:59Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (6af872f48e9930297eb9b7408690ad0524d9893461a72a1bd61fde0594914d2d) config message + 2023-08-15T01:40:59Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40948: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40950: read tcp 127.0.0.1:46751->127.0.0.1:40950: read: connection reset by peer + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40956: read tcp 127.0.0.1:46751->127.0.0.1:40956: read: connection reset by peer + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40968: read tcp 127.0.0.1:46751->127.0.0.1:40968: read: connection reset by peer + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:41012: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40996: EOF + 2023/08/15 01:40:59 http: TLS handshake error from 127.0.0.1:40984: read tcp 127.0.0.1:46751->127.0.0.1:40984: read: connection reset by peer + 2023-08-15T01:40:59Z INFO plugin.alerting successfully synced (6/6) conditions {"component": "alarms"} + 2023-08-15T01:40:59Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:356502993 {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:119253770 {"component": "alarms", "onSystemConditionCreate": "bGpeDG5QAFzsEZB5R9j8kk"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:119253770 {"component": "alarms", "onSystemConditionCreate": "bGpeDG5QAFzsEZB5R9j8kk"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:356502993 {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:119253770 {"component": "alarms", "onSystemConditionCreate": "dCfr8i4Q6EHPvxZwW9gLt6"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:356502993 {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:119253770 {"component": "alarms", "onSystemConditionCreate": "dCfr8i4Q6EHPvxZwW9gLt6"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:119253770 {"component": "alarms", "onSystemConditionCreate": "bGpeDG5QAFzsEZB5R9j8kk"} + 2023-08-15T01:40:59Z DEBUG plugin.alerting received agent health update connected true : seconds:1692063658 nanos:119253770 {"component": "alarms", "onSystemConditionCreate": "dCfr8i4Q6EHPvxZwW9gLt6"} +• [0.974 seconds] +------------------------------ +Invalidated and clean up suite test when we delete an alarm of the metrics type should clean up the alarm configuration itself [integration] +/home/ec2-user/opni/test/plugins/alerting/delete_test.go:273 +• [0.004 seconds] +------------------------------ +Invalidated and clean up suite test when We uninstall metrics capability from a cluster that has metrics alarms should switch these alarms to the invalidated state [integration] +/home/ec2-user/opni/test/plugins/alerting/delete_test.go:293 + 2023-08-15T01:41:00Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["metrics"]} + 2023-08-15T01:41:00Z DEBUG plugin.alerting ignoring sync request due to capability filter 'alerting' {"component": "health-cfg-sync"} + 2023-08-15T01:41:00Z DEBUG plugin.metrics received sync request + 2023-08-15T01:41:00Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:41:00Z INFO plugin.metrics.metrics-backend node sync requested {"cluster": "agent1", "capability": "metrics"} + 2023-08-15T01:41:00Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent1", "time": "2023-08-15T01:41:00Z"} + 2023-08-15T01:41:00Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:41:00Z DEBUG test.env configuring node {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:41:00Z INFO test.env stopping prometheus {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:41:00Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent1"} + 2023-08-15T01:41:00Z DEBUG plugin.metrics stopping rule sync + 2023-08-15T01:41:00Z DEBUG plugin.metrics disabling http server + 2023-08-15T01:41:00Z WARN plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:41:00Z DEBUG test.env configuring node {"node": "agent1", "driver": "otel"} + 2023-08-15T01:41:00Z DEBUG plugin.metrics rule discovery stream closing {"error": "context canceled"} + 2023-08-15T01:41:00Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "otel"} + 2023-08-15T01:41:00Z INFO tasks Stored data will not be deleted {"id": "agent1"} + 2023-08-15T01:41:00Z INFO tasks Uninstalling metrics capability for this cluster {"id": "agent1"} + 2023-08-15T01:41:00Z INFO tasks Time series data will not be deleted {"id": "agent1"} +• [0.034 seconds] +------------------------------ + 2023-08-15T01:41:00Z INFO tasks Removing capability from cluster metadata {"id": "agent1"} +Invalidated and clean up suite test when we delete an alarm of the internal type should successfully submit the alarm for deletion [integration] +/home/ec2-user/opni/test/plugins/alerting/delete_test.go:327 +• [0.005 seconds] +------------------------------ +Invalidated and clean up suite test when we delete an alarm of the internal type should clean up the alarm configuration itself [integration] +/home/ec2-user/opni/test/plugins/alerting/delete_test.go:334 + 2023-08-15T01:41:00Z INFO tasks Capability uninstalled successfully {"id": "agent1"} + 2023-08-15T01:41:00Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:00Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:00Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:00Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:00Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:00Z INFO plugin.alerting syncing 5 conditions {"component": "alarms"} + 2023-08-15T01:41:00Z INFO plugin.alerting evaluation context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "bGpeDG5QAFzsEZB5R9j8kk"} + 2023-08-15T01:41:00Z INFO plugin.alerting evaluation context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "bGpeDG5QAFzsEZB5R9j8kk"} + 2023-08-15T01:41:00Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:41:00Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "dCfr8i4Q6EHPvxZwW9gLt6"} + 2023-08-15T01:41:00Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:41:00Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "b1d328c9-a063-4e6f-993d-10490e6e76fb"} + 2023-08-15T01:41:00Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "dCfr8i4Q6EHPvxZwW9gLt6"} + 2023-08-15T01:41:00Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "f381c452-12ec-4c96-b432-54636f604a96"} + 2023-08-15T01:41:00Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onCapabilityStatusCreate": "stdRzQww4KPXDy5r3dVAh6"} + 2023-08-15T01:41:00Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onCapabilityStatusCreate": "FsECjbp6G7xntHLX5o9DjR"} + 2023-08-15T01:41:00Z INFO plugin.alerting shutting down cluster driver update handler + 2023-08-15T01:41:00Z DEBUG plugin.alerting closing cortex cluster status watcher... {"watcher": "cortex-cluster-status"} + 2023-08-15T01:41:00Z DEBUG gateway.sync agent disconnected {"id": "agent2"} + 2023-08-15T01:41:00Z DEBUG gateway.delegate agent disconnected {"id": "agent2"} + 2023-08-15T01:41:00Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:41:00Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:00Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:00Z INFO gateway shutting down plugins + 2023-08-15T01:41:00Z WARN plugin.alerting failed to receive cluster health status from grpc stream, retrying... + ts=2023-08-15T01:41:00.881Z caller=alertmanager_main.go:601 level=info msg="Received SIGTERM, exiting gracefully..." + ts=2023-08-15T01:41:00.881Z caller=cluster.go:690 level=info component=cluster msg="gossip not settled but continuing anyway" polls=1 elapsed=3.815512384s + 2023-08-15T01:41:00Z DEBUG plugin.alerting received condition update: name:"agent-disconnect" description:"Alert when the downstream agent disconnects from the opni upstream" labels:"agent-disconnect" labels:"opni" labels:"_default" severity:Critical alertType:{system:{clusterId:{id:"agent2"} timeout:{seconds:1}}} lastUpdated:{seconds:1692063658 nanos:297438417} id:"34e6zCJWGxLSxNSP97cftd" metadata:{key:"opni.io/alarm-hash" value:"4029a2c21ef580bbc9f2eaf1add48fb7caadc895c9af7002c2cb5678775bb7a0"} {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:41:00Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error + 2023-08-15T01:41:00Z DEBUG gateway.sync agent disconnected {"id": "agent1"} + 2023-08-15T01:41:00Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:41:00Z DEBUG gateway.delegate agent disconnected {"id": "agent1"} + 2023-08-15T01:41:00Z DEBUG plugin.alerting Creating agent disconnect with timeout 1s {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:41:00Z WARN jetstream disconnected from jetstream +write tcp 127.0.0.1:49356->127.0.0.1:42403: write: connection reset by peer on connection [37] + 2023-08-15T01:41:00Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z INFO gateway all plugins shut down + 2023-08-15T01:41:00Z DEBUG plugin.alerting received condition update: name:"agent-capability-unhealthy" description:"Alert when some downstream agent capability becomes unhealthy" labels:"agent-capability-health" labels:"opni" labels:"_default" severity:Critical alertType:{downstreamCapability:{clusterId:{id:"agent2"} capabilityState:"Failure" capabilityState:"Pending" for:{seconds:1}}} lastUpdated:{seconds:1692063658 nanos:302402760} id:"stdRzQww4KPXDy5r3dVAh6" metadata:{key:"opni.io/alarm-hash" value:"5f135eda476394945cbeaf43e3b102d3cbf5a0861e357ae3342ee6fcb86f5d55"} {"component": "alarms", "onCapabilityStatusCreate": "stdRzQww4KPXDy5r3dVAh6"} + 2023-08-15T01:41:00Z DEBUG plugin.alerting Creating agent capability unhealthy with timeout 1s {"component": "alarms", "onCapabilityStatusCreate": "stdRzQww4KPXDy5r3dVAh6"} +write tcp 127.0.0.1:49356->127.0.0.1:42403: write: broken pipe on connection [37] + 2023-08-15T01:41:00Z WARN gateway http server exited with error {"error": "context canceled"} + 2023-08-15T01:41:00Z WARN test.env gateway server exited with error {"error": "context canceled"} + 2023-08-15T01:41:00Z WARN plugin.alerting-syncer syncer/syncer_server.go:163 remote syncer unavailable, reconnecting, ... + 2023-08-15T01:41:00Z WARN test.env management server exited with error {"error": "context canceled"} + 2023-08-15T01:41:00Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "dada5d033f242ec39f88a12fdcc651db58bcfd009f2f1e5aa740ac523f4034bd"} + 2023-08-15T01:41:00Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z ERROR plugin.alerting EOF + 2023-08-15T01:41:00Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:41:00Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z INFO test.env Cortex stopping... + 2023-08-15T01:41:00Z ERROR plugin.alerting failed to receive cluster event : rpc error: code = Unavailable desc = error reading from server: read unix @->/tmp/plugin1670833980: use of closed network connection + 2023-08-15T01:41:00Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z ERROR test agent "agent2" exited: context canceled + 2023-08-15T01:41:00Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:41:00Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:00Z ERROR test agent "agent1" exited: context canceled + 2023-08-15T01:41:00Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error {"error": "rpc error: code = Canceled desc = context canceled"} +• [0.721 seconds] +------------------------------ +Node Config should initially have all nodes using the default config [integration] +/home/ec2-user/opni/test/plugins/alerting/node_config_test.go:122 + 2023-08-15T01:41:00Z INFO test.env Starting test environment + 2023-08-15T01:41:00Z INFO test.env Copying 7 files from embedded testdata/cortex to /tmp/opni-test-4032888549/cortex + 2023-08-15T01:41:00Z INFO test.env Waiting for jetstream to start... + 2023-08-15T01:41:00Z INFO test.env Cortex stopped + [231696] 2023/08/15 01:41:00.905057 [INF] Starting nats-server + [231696] 2023/08/15 01:41:00.905101 [INF] Version: 2.9.21 + [231696] 2023/08/15 01:41:00.905106 [INF] Git: [b2e7725] + [231696] 2023/08/15 01:41:00.905109 [INF] Name: NDUJMJCTVOXVPMTYVHH2W4VPQT6Q42NVYV2OALV44OCWIHI25NDRXJA3 + [231696] 2023/08/15 01:41:00.905115 [INF] Node: W4euMW8u + [231696] 2023/08/15 01:41:00.905119 [INF] ID: NDUJMJCTVOXVPMTYVHH2W4VPQT6Q42NVYV2OALV44OCWIHI25NDRXJA3 + [231696] 2023/08/15 01:41:00.905126 [INF] Using configuration file: /tmp/opni-test-4032888549/jetstream/jetstream.conf + [231696] 2023/08/15 01:41:00.905621 [INF] Starting JetStream + [231696] 2023/08/15 01:41:00.905794 [INF] _ ___ _____ ___ _____ ___ ___ _ __ __ + [231696] 2023/08/15 01:41:00.905808 [INF] _ | | __|_ _/ __|_ _| _ \ __| /_\ | \/ | + [231696] 2023/08/15 01:41:00.905812 [INF] | || | _| | | \__ \ | | | / _| / _ \| |\/| | + [231696] 2023/08/15 01:41:00.905815 [INF] \__/|___| |_| |___/ |_| |_|_\___/_/ \_\_| |_| + [231696] 2023/08/15 01:41:00.905818 [INF] + [231696] 2023/08/15 01:41:00.905822 [INF] https://docs.nats.io/jetstream + [231696] 2023/08/15 01:41:00.905825 [INF] + [231696] 2023/08/15 01:41:00.905828 [INF] ---------------- JETSTREAM ---------------- + [231696] 2023/08/15 01:41:00.905833 [INF] Max Memory: 11.71 GB + [231696] 2023/08/15 01:41:00.905837 [INF] Max Storage: 5.29 GB + [231696] 2023/08/15 01:41:00.905841 [INF] Store Directory: "/tmp/opni-test-4032888549/jetstream/data/jetstream" + [231696] 2023/08/15 01:41:00.905843 [INF] ------------------------------------------- + [231696] 2023/08/15 01:41:00.906224 [INF] Listening for client connections on 0.0.0.0:44491 + [231696] 2023/08/15 01:41:00.906543 [INF] Server is ready + 2023-08-15T01:41:00Z INFO test.env Jetstream started + 2023-08-15T01:41:00Z DEBUG keyring loaded ephemeral key {"path": "/tmp/opni-test-4032888549/keyring/local-agent.json", "usage": "auth", "labels": {"opni.io/session-attribute":"local"}} + 2023-08-15T01:41:00Z DEBUG gateway found 0 plugins + 2023-08-15T01:41:00Z DEBUG gateway loaded plugin manifest {"plugins": 0} + 2023-08-15T01:41:00Z INFO gateway.cache compressing and archiving plugins... + 2023-08-15T01:41:00Z DEBUG gateway.cache added 0 new plugins to cache + 2023-08-15T01:41:00Z INFO gateway running plugin cache gc + 2023-08-15T01:41:00Z INFO gateway.update-server registering update handler for strategy "binary" + 2023-08-15T01:41:00Z INFO gateway.update-server registering update handler for strategy "kubernetes" + 2023-08-15T01:41:00Z INFO gateway.update-server registering update handler for strategy "noop" + 2023-08-15T01:41:00Z DEBUG gateway.grpc registering service {"service": "control.HealthListener"} + 2023-08-15T01:41:00Z DEBUG gateway.grpc registering internal service {"service": "stream.Delegate"} + 2023-08-15T01:41:00Z DEBUG gateway.grpc registering service {"service": "core.Pinger"} + 2023-08-15T01:41:00Z INFO test.env Loading gateway plugins... + 2023-08-15T01:41:00Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:00Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:00Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.backends.Capability", "opni.Metrics"]} + 2023-08-15T01:41:00Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.System", "opni.apiextensions.HTTPAPIExtension", "opni.apiextensions.ManagementAPIExtension", "opni.Metrics", "opni.backends.Capability", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.ManagementAPIExtension"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Metrics"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Metrics"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:00Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/metrics", "capability": "metrics"} + 2023-08-15T01:41:00Z INFO mgmt got extension descriptor for service cortexadmin.CortexAdmin + 2023-08-15T01:41:00Z INFO mgmt got extension descriptor for service alerting.AlertConditions + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.System"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:00Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_alerting"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:00Z INFO gateway added capability backend {"plugin": "github.com/rancher/opni/plugins/alerting", "capability": "alerting"} + 2023-08-15T01:41:00Z DEBUG gateway.grpc plugin rejected internal stream connection {"plugin": "plugin_alerting"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:00Z DEBUG gateway.grpc connecting to gateway plugin {"plugin": "plugin_metrics"} + 2023-08-15T01:41:00Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.System"} + 2023-08-15T01:41:00Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:00Z INFO mgmt loading service {"name": "cortexadmin.CortexAdmin"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/AllUserStats"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/WriteMetrics"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/Query"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/QueryRange"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetRule"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricMetadata"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ListRules"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/LoadRules"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/DeleteRule"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/FlushBlocks"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetSeriesMetrics"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetMetricLabelSets"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexStatus"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/GetCortexConfig"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/cortexadmin.CortexAdmin/ExtractRawSeries"} + 2023-08-15T01:41:00Z INFO mgmt loading http rules {"name": "cortexadmin.CortexAdmin", "rules": 17} + 2023-08-15T01:41:00Z DEBUG mgmt rule descriptors {"name": "cortexadmin.CortexAdmin", "rules": [{"http":{"Pattern":{"Get":"/all_user_stats"}},"method":{"name":"AllUserStats","input_type":".google.protobuf.Empty","output_type":".cortexadmin.UserIDStatsList","options":{}}},{"http":{"Pattern":{"Post":"/write_metrics"},"body":"*"},"method":{"name":"WriteMetrics","input_type":".cortexadmin.WriteRequest","output_type":".cortexadmin.WriteResponse","options":{}}},{"http":{"Pattern":{"Get":"/query"},"additional_bindings":[{"Pattern":{"Post":"/query"},"body":"*"}]},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query"},"body":"*"},"method":{"name":"Query","input_type":".cortexadmin.QueryRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/query_range"},"additional_bindings":[{"Pattern":{"Post":"/query_range"},"body":"*"}]},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Post":"/query_range"},"body":"*"},"method":{"name":"QueryRange","input_type":".cortexadmin.QueryRangeRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/rules/{namespace}/{groupName}"}},"method":{"name":"GetRule","input_type":".cortexadmin.GetRuleRequest","output_type":".cortexadmin.QueryResponse","options":{}}},{"http":{"Pattern":{"Get":"/metadata"}},"method":{"name":"GetMetricMetadata","input_type":".cortexadmin.MetricMetadataRequest","output_type":".cortexadmin.MetricMetadata","options":{}}},{"http":{"Pattern":{"Get":"/rules"}},"method":{"name":"ListRules","input_type":".cortexadmin.ListRulesRequest","output_type":".cortexadmin.ListRulesResponse","options":{}}},{"http":{"Pattern":{"Post":"/rules"}},"method":{"name":"LoadRules","input_type":".cortexadmin.LoadRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/rules/{groupName}"}},"method":{"name":"DeleteRule","input_type":".cortexadmin.DeleteRuleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/flush_blocks"}},"method":{"name":"FlushBlocks","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/series/metadata"}},"method":{"name":"GetSeriesMetrics","input_type":".cortexadmin.SeriesRequest","output_type":".cortexadmin.SeriesInfoList","options":{}}},{"http":{"Pattern":{"Get":"/series/labels"}},"method":{"name":"GetMetricLabelSets","input_type":".cortexadmin.LabelRequest","output_type":".cortexadmin.MetricLabels","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetCortexStatus","input_type":".google.protobuf.Empty","output_type":".cortexadmin.CortexStatus","options":{}}},{"http":{"Pattern":{"Get":"/config"}},"method":{"name":"GetCortexConfig","input_type":".cortexadmin.ConfigRequest","output_type":".cortexadmin.ConfigResponse","options":{}}},{"http":{"Pattern":{"Get":"/series/raw"}},"method":{"name":"ExtractRawSeries","input_type":".cortexadmin.MatcherRequest","output_type":".cortexadmin.QueryResponse","options":{}}}]} + 2023-08-15T01:41:00Z INFO mgmt got extension descriptor for service cortexops.CortexOps + 2023-08-15T01:41:00Z DEBUG plugin.metrics/pkg/gateway.stream internal gateway stream connected + 2023-08-15T01:41:00Z INFO mgmt loading service {"name": "alerting.AlertConditions"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionGroups"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CreateAlertCondition"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/GetAlertCondition"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditions"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/UpdateAlertCondition"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionChoices"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeleteAlertCondition"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/AlertConditionStatus"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ListAlertConditionsWithStatus"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/CloneTo"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/ActivateSilence"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/DeactivateSilence"} + 2023-08-15T01:41:00Z INFO mgmt loading method {"name": "/alerting.AlertConditions/Timeline"} + 2023-08-15T01:41:00Z INFO mgmt loading http rules {"name": "alerting.AlertConditions", "rules": 13} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "alerting.AlertConditions", "rules": [{"http":{"Pattern":{"Get":"/groups"}},"method":{"name":"ListAlertConditionGroups","input_type":".google.protobuf.Empty","output_type":".core.ReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertCondition","input_type":".alerting.AlertCondition","output_type":".alerting.ConditionReference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertCondition","input_type":".alerting.ConditionReference","output_type":".alerting.AlertCondition","options":{}}},{"http":{"Pattern":{"Post":"/list"},"body":"*"},"method":{"name":"ListAlertConditions","input_type":".alerting.ListAlertConditionRequest","output_type":".alerting.AlertConditionList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertCondition","input_type":".alerting.UpdateAlertConditionRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/choices"}},"method":{"name":"ListAlertConditionChoices","input_type":".alerting.AlertDetailChoicesRequest","output_type":".alerting.ListAlertTypeDetails","options":{}}},{"http":{"Pattern":{"Delete":"/configure"}},"method":{"name":"DeleteAlertCondition","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status/{id}"}},"method":{"name":"AlertConditionStatus","input_type":".alerting.ConditionReference","output_type":".alerting.AlertStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/list/withStatus"},"body":"*"},"method":{"name":"ListAlertConditionsWithStatus","input_type":".alerting.ListStatusRequest","output_type":".alerting.ListStatusResponse","options":{}}},{"http":{"Pattern":{"Post":"/clone"},"body":"*"},"method":{"name":"CloneTo","input_type":".alerting.CloneToRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/silences"},"body":"*"},"method":{"name":"ActivateSilence","input_type":".alerting.SilenceRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/silences"}},"method":{"name":"DeactivateSilence","input_type":".alerting.ConditionReference","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/timeline"}},"method":{"name":"Timeline","input_type":".alerting.TimelineRequest","output_type":".alerting.TimelineResponse","options":{}}}]} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service alerting.AlertEndpoints + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "alerting.AlertEndpoints"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/CreateAlertEndpoint"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/GetAlertEndpoint"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ToggleNotifications"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/ListAlertEndpoints"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/UpdateAlertEndpoint"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/DeleteAlertEndpoint"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertEndpoints/TestAlertEndpoint"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "alerting.AlertEndpoints", "rules": 7} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "alerting.AlertEndpoints", "rules": [{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"CreateAlertEndpoint","input_type":".alerting.AlertEndpoint","output_type":".core.Reference","options":{}}},{"http":{"Pattern":{"Post":"/list/{id}"},"body":"*"},"method":{"name":"GetAlertEndpoint","input_type":".core.Reference","output_type":".alerting.AlertEndpoint","options":{}}},{"http":{"Pattern":{"Post":"/toggleNotifications"},"body":"*"},"method":{"name":"ToggleNotifications","input_type":".alerting.ToggleRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/list"}},"method":{"name":"ListAlertEndpoints","input_type":".alerting.ListAlertEndpointsRequest","output_type":".alerting.AlertEndpointList","options":{}}},{"http":{"Pattern":{"Put":"/configure"},"body":"*"},"method":{"name":"UpdateAlertEndpoint","input_type":".alerting.UpdateAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/delete"},"body":"*"},"method":{"name":"DeleteAlertEndpoint","input_type":".alerting.DeleteAlertEndpointRequest","output_type":".alerting.ConditionReferenceList","options":{}}},{"http":{"Pattern":{"Post":"/test"},"body":"*"},"method":{"name":"TestAlertEndpoint","input_type":".alerting.TestAlertEndpointRequest","output_type":".alerting.TestAlertEndpointResponse","options":{}}}]} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service alerting.AlertNotifications + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "alerting.AlertNotifications"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/TriggerAlerts"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ResolveAlerts"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/PushNotification"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListNotifications"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListAlarmMessages"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.AlertNotifications/ListRoutingRelationships"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "alerting.AlertNotifications", "rules": 4} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "alerting.AlertNotifications", "rules": [{"http":{"Pattern":{"Post":"/trigger"},"body":"*"},"method":{"name":"TriggerAlerts","input_type":".alerting.TriggerAlertsRequest","output_type":".alerting.TriggerAlertsResponse","options":{}}},{"http":{"Pattern":{"Post":"/push"},"body":"*"},"method":{"name":"PushNotification","input_type":".alerting.Notification","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/notifications/list"},"body":"*"},"method":{"name":"ListNotifications","input_type":".alerting.ListNotificationRequest","output_type":".alerting.ListMessageResponse","options":{}}},{"http":{"Pattern":{"Post":"/alarms/list"},"body":"*"},"method":{"name":"ListAlarmMessages","input_type":".alerting.ListAlarmMessageRequest","output_type":".alerting.ListMessageResponse","options":{}}}]} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service alerting.ops.AlertingAdmin + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (8) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "cortexops.CortexOps"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/cortexops.CortexOps/ConfigureCluster"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/cortexops.CortexOps/GetClusterStatus"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/cortexops.CortexOps/UninstallCluster"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "cortexops.CortexOps", "rules": 4} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "cortexops.CortexOps", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".cortexops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".cortexops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".cortexops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service remoteread.RemoteReadGateway + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "alerting.ops.AlertingAdmin"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterConfiguration"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/trace", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/profile", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/block", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/heap", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_alerting/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/ready", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /plugin_alerting/healthy", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_alerting/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PUT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PATCH /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "HEAD /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v2/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "TRACE /plugin_alerting/alertmanager/api/v1/*any", "plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/ConfigureCluster"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/GetClusterStatus"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/InstallCluster"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/UninstallCluster"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.ops.AlertingAdmin/Info"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "alerting.ops.AlertingAdmin", "rules": 6} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "alerting.ops.AlertingAdmin", "rules": [{"http":{"Pattern":{"Get":"/configuration"}},"method":{"name":"GetClusterConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ClusterConfiguration","options":{}}},{"http":{"Pattern":{"Post":"/configure"},"body":"*"},"method":{"name":"ConfigureCluster","input_type":".alerting.ops.ClusterConfiguration","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/status"}},"method":{"name":"GetClusterStatus","input_type":".google.protobuf.Empty","output_type":".alerting.ops.InstallStatus","options":{}}},{"http":{"Pattern":{"Post":"/install"}},"method":{"name":"InstallCluster","input_type":".google.protobuf.Empty","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/uninstall"}},"method":{"name":"UninstallCluster","input_type":".alerting.ops.UninstallRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/info"}},"method":{"name":"Info","input_type":".google.protobuf.Empty","output_type":".alerting.ops.ComponentInfo","options":{}}}]} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service alerting.ops.ConfigReconciler + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.ops.ConfigReconciler/SyncConfig"} + 2023-08-15T01:41:01Z INFO mgmt service has no http rules {"name": "alerting.ops.ConfigReconciler"} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service alerting.node.AlertingNodeConfiguration + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "remoteread.RemoteReadGateway"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/AddTarget"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/EditTarget"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/RemoveTarget"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/ListTargets"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Start"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Stop"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/GetTargetStatus"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/remoteread.RemoteReadGateway/Discover"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "remoteread.RemoteReadGateway", "rules": 8} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "remoteread.RemoteReadGateway", "rules": [{"http":{"Pattern":{"Post":"/targets"},"body":"*"},"method":{"name":"AddTarget","input_type":".remoteread.TargetAddRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Put":"/targets"},"body":"*"},"method":{"name":"EditTarget","input_type":".remoteread.TargetEditRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Delete":"/targets/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"RemoveTarget","input_type":".remoteread.TargetRemoveRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/targets"}},"method":{"name":"ListTargets","input_type":".remoteread.TargetListRequest","output_type":".remoteread.TargetList","options":{}}},{"http":{"Pattern":{"Post":"/targets/start/{target.meta.clusterId}/{target.meta.name}"},"body":"*"},"method":{"name":"Start","input_type":".remoteread.StartReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/targets/stop/{meta.clusterId}/{meta.name}"},"body":"*"},"method":{"name":"Stop","input_type":".remoteread.StopReadRequest","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Post":"/status"},"body":"*"},"method":{"name":"GetTargetStatus","input_type":".remoteread.TargetStatusRequest","output_type":".remoteread.TargetStatus","options":{}}},{"http":{"Pattern":{"Post":"/edit"},"body":"*"},"method":{"name":"Discover","input_type":".remoteread.DiscoveryRequest","output_type":".remoteread.DiscoveryResponse","options":{}}}]} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service node.metrics.NodeConfiguration + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "alerting.node.AlertingNodeConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.node.AlertingNodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "alerting.node.AlertingNodeConfiguration", "rules": 4} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "alerting.node.AlertingNodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".alerting.node.AlertingCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".alerting.node.AlertingCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".alerting.node.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:41:01Z INFO mgmt got extension descriptor for service alerting.node.NodeAlertingCapability + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "alerting.node.NodeAlertingCapability"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/alerting.node.NodeAlertingCapability/Sync"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "alerting.node.NodeAlertingCapability", "rules": 1} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "alerting.node.NodeAlertingCapability", "rules": [{"http":{"Pattern":{"Post":"/v1/node/sync"},"body":"*"},"method":{"name":"Sync","input_type":".alerting.node.AlertingCapabilityConfig","output_type":".alerting.node.SyncResponse","options":{}}}]} + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/gateway.stream calling client handler + 2023-08-15T01:41:01Z INFO mgmt loading service {"name": "node.metrics.NodeConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetDefaultConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetDefaultConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/GetNodeConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading method {"name": "/node.metrics.NodeConfiguration/SetNodeConfiguration"} + 2023-08-15T01:41:01Z INFO mgmt loading http rules {"name": "node.metrics.NodeConfiguration", "rules": 4} + 2023-08-15T01:41:01Z DEBUG mgmt rule descriptors {"name": "node.metrics.NodeConfiguration", "rules": [{"http":{"Pattern":{"Get":"/node_config"}},"method":{"name":"GetDefaultConfiguration","input_type":".google.protobuf.Empty","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config"},"body":"*"},"method":{"name":"SetDefaultConfiguration","input_type":".node.metrics.MetricsCapabilitySpec","output_type":".google.protobuf.Empty","options":{}}},{"http":{"Pattern":{"Get":"/node_config/{id}"}},"method":{"name":"GetNodeConfiguration","input_type":".core.Reference","output_type":".node.metrics.MetricsCapabilitySpec","options":{}}},{"http":{"Pattern":{"Put":"/node_config/{node.id}"},"body":"*"},"method":{"name":"SetNodeConfiguration","input_type":".node.metrics.NodeConfigRequest","output_type":".google.protobuf.Empty","options":{}}}]} + 2023-08-15T01:41:01Z INFO plugin.metrics.cortex-http configuring http api server + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/label/:name/values", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z INFO plugin.alerting using cluster driver {"driver": "test-environment"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /prometheus/api/v1/metadata", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /ready", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "GET /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_range", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/query_exemplars", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z INFO plugin.opni.alerting starting opni embedded server {"addr": "127.0.0.1:45261"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/read", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/labels", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PUT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PUT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PATCH /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "PATCH /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "HEAD /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "HEAD /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "OPTIONS /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "DELETE /prometheus/api/v1/series", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "CONNECT /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/alerts", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/v1/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "TRACE /api/prom/rules", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG gateway.http configured route for plugin {"route": "TRACE /multitenant_alertmanager", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z INFO test.env Waiting for gateway to start... + 2023-08-15T01:41:01Z INFO test.env loaded 12 plugins + 2023-08-15T01:41:01Z INFO mgmt management HTTP server starting {"address": "localhost:42285"} + 2023-08-15T01:41:01Z INFO mgmt management gRPC server starting {"address": "127.0.0.1:43185"} + 2023-08-15T01:41:01Z INFO gateway.grpc gateway gRPC server starting {"address": "127.0.0.1:46393"} + 2023-08-15T01:41:01Z INFO gateway.http gateway HTTP server starting {"api": "127.0.0.1:45035", "metrics": "127.0.0.1:35949"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/tokens"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/tokens/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/tokens/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/watch"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/clusters/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/certs"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{id}/health"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/health/watch"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/clusters/{cluster.id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/roles"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/roles"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/roles/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/rolebindings"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/rolebindings"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/roles"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/rolebindings"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/subjectaccess"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/apiextensions"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/config"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/config"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/capabilities"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/capabilities/{name}/installer"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/install"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{target.cluster.id}/capabilities/{name}/uninstall"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/status"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/status"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/Management/clusters/{cluster.id}/capabilities/{name}/uninstall/cancel"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/Management/dashboard/settings"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/Management/dashboard/settings"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/all_user_stats"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/write_metrics"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/query_range"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules/{namespace}/{groupName}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/metadata"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/rules"} + 2023-08-15T01:41:01Z INFO test.env Gateway started + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/rules"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/CortexAdmin/rules/{groupName}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexAdmin/flush_blocks"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/metadata"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/labels"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/status"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/config"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexAdmin/series/raw"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertConditions/groups"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/configure"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertConditions/configure"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/choices"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/configure"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/status/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/list/withStatus"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/clone"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/silences"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/AlertConditions/silences"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertConditions/timeline"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/list/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/toggleNotifications"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertEndpoints/list"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertEndpoints/configure"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/delete"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertEndpoints/test"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/trigger"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/push"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/notifications/list"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertNotifications/alarms/list"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/configuration"} + 2023-08-15T01:41:01Z INFO test starting agent {"id": "agent1", "address": "127.0.0.1:43839", "version": "v2"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/configure"} + 2023-08-15T01:41:01Z DEBUG agent using log level: debug + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/CortexOps/status"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/CortexOps/uninstall"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/configuration"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/configure"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/status"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/install"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/AlertingAdmin/uninstall"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingAdmin/info"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "DELETE", "path": "/RemoteReadGateway/targets/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/RemoteReadGateway/targets"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/start/{target.meta.clusterId}/{target.meta.name}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/targets/stop/{meta.clusterId}/{meta.name}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/status"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/RemoteReadGateway/edit"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/AlertingNodeConfiguration/node_config/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/AlertingNodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "POST", "path": "/NodeAlertingCapability/v1/node/sync"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "GET", "path": "/NodeConfiguration/node_config/{id}"} + 2023-08-15T01:41:01Z DEBUG mgmt configured http handler {"method": "PUT", "path": "/NodeConfiguration/node_config/{node.id}"} + 2023-08-15T01:41:01Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:41:01Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent1 + 2023-08-15T01:41:01Z INFO agent bootstrap completed successfully + 2023-08-15T01:41:01Z DEBUG keyring loaded ephemeral key {"path": "/tmp/opni-test-4032888549/keyring/local-agent.json", "usage": "auth", "labels": {"opni.io/session-attribute":"local"}} + 2023-08-15T01:41:01Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:01Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:41:01Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:41:01Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent1 is now active + 2023-08-15T01:41:01Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:41:01Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension", "opni.backends.Capability", "opni.Health"]} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:41:01Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:41:01Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:41:01Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:41:01Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:41:01Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:41:01Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:41:01Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:41:01Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:41:01Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:41:01Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:41:01Z INFO agent using unmanaged plugin loader + 2023-08-15T01:41:01Z INFO agent agent http server starting {"address": "127.0.0.1:43839"} + 2023-08-15T01:41:01Z INFO agent connecting to gateway... + 2023-08-15T01:41:01Z DEBUG gateway ratelimit: 50.000000 available + 2023-08-15T01:41:01Z INFO test starting agent {"id": "agent2", "address": "127.0.0.1:44837", "version": "v2"} + 2023-08-15T01:41:01Z DEBUG agent using log level: debug + 2023-08-15T01:41:01Z DEBUG gateway-client authenticated {"id": "agent1", "attributes": ["local"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:41:01Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent1"} + 2023-08-15T01:41:01Z INFO agent no existing keyring found, starting bootstrap process + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent1"} + 2023-08-15T01:41:01Z DEBUG gateway.sync agent connected {"id": "agent1"} + 2023-08-15T01:41:01Z INFO gateway.monitor received status update {"id": "agent1", "connected": true, "attributes": ["local"]} + 2023-08-15T01:41:01Z DEBUG gateway.delegate agent connected {"id": "agent1"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:41:01Z INFO gateway.monitor received health update {"id": "agent1", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:41:01Z INFO agent gateway connected + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:41:01Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:41:01Z INFO agent bootstrap completed successfully + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z INFO plugin.alerting added durable ordered push consumer for cluster %sagent2 + 2023-08-15T01:41:01Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:41:01Z DEBUG plugin.alerting could not find driver : k8s_driver + 2023-08-15T01:41:01Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:01Z INFO pluginloader loading plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:41:01Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:41:01Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:41:01Z INFO gateway.monitor received health update {"id": "agent1", "ready": true, "conditions": []} + 2023-08-15T01:41:01Z DEBUG test.env configuring node {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:41:01Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "prometheus"} + 2023-08-15T01:41:01Z DEBUG test.env configuring node {"node": "agent1", "driver": "otel"} + 2023-08-15T01:41:01Z DEBUG test.env nothing to do (already stopped) {"node": "agent1", "driver": "otel"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent1"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:41:01Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/metrics", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension", "opni.apiextensions.HTTPAPIExtension"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.backends.Capability"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:41:01Z DEBUG pluginloader checking if plugin implements any interfaces in the scheme {"plugin": "github.com/rancher/opni/plugins/alerting", "interfaces": ["opni.backends.Capability", "opni.Health", "opni.apiextensions.StreamAPIExtension"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting downstream agent disconnect condition on cluster creation for cluster agent2 is now active + 2023-08-15T01:41:01Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/metrics + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.Health"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.backends.Capability"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z INFO agent loaded capability node plugin github.com/rancher/opni/plugins/alerting + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.Health"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent1"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/metrics", "id": "opni.apiextensions.HTTPAPIExtension"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG pluginloader implementation found {"plugin": "github.com/rancher/opni/plugins/alerting", "id": "opni.apiextensions.StreamAPIExtension"} + 2023-08-15T01:41:01Z DEBUG pluginloader invoking load hooks (4) {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG agent loaded stream api extension plugin {"plugin": "github.com/rancher/opni/plugins/alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "POST /api/agent/push", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "POST /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/trace", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/threadcreate", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/cmdline", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/profile", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/symbol", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/allocs", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/block", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/goroutine", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/heap", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent configured route for plugin {"route": "GET /debug/plugin_metrics/pprof/mutex", "plugin": "github.com/rancher/opni/plugins/metrics"} + 2023-08-15T01:41:01Z DEBUG agent.agent-updater sending manifest sync request {"type": "agent", "entries": 1} + 2023-08-15T01:41:01Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:41:01Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:41:01Z INFO agent.agent-updater received sync response {"type": "agent"} + 2023-08-15T01:41:01Z INFO agent.agent-updater manifest sync complete {"type": "agent", "entries": 1} + 2023-08-15T01:41:01Z DEBUG agent.plugin-updater sending manifest sync request {"type": "plugin", "entries": 1} + 2023-08-15T01:41:01Z INFO gateway.update-server syncing agent manifest {"strategy": "noop"} + 2023-08-15T01:41:01Z INFO gateway.update-server computed updates {"patches": 1} + 2023-08-15T01:41:01Z INFO agent.plugin-updater received sync response {"type": "plugin"} + 2023-08-15T01:41:01Z INFO agent.plugin-updater manifest sync complete {"type": "plugin", "entries": 1} + 2023-08-15T01:41:01Z INFO agent using unmanaged plugin loader + 2023-08-15T01:41:01Z INFO agent agent http server starting {"address": "127.0.0.1:44837"} + 2023-08-15T01:41:01Z INFO agent connecting to gateway... + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:41:01Z DEBUG gateway ratelimit: 49.998515 available + 2023-08-15T01:41:01Z DEBUG gateway-client authenticated {"id": "agent2"} + 2023-08-15T01:41:01Z DEBUG gateway.grpc handling new stream connection + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream stream connected + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/alerting.stream stream connected {"id": "agent2"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream stream connected + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/gateway.stream stream connected {"id": "agent2"} +• [0.346 seconds] +------------------------------ +Node Config when changing the default config should return the new config for all nodes [integration] +/home/ec2-user/opni/test/plugins/alerting/node_config_test.go:149 + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/alerting.stream stream server started + 2023-08-15T01:41:01Z INFO gateway.monitor received status update {"id": "agent2", "connected": true} + 2023-08-15T01:41:01Z DEBUG gateway.sync agent connected {"id": "agent2"} + 2023-08-15T01:41:01Z DEBUG gateway.delegate agent connected {"id": "agent2"} + 2023-08-15T01:41:01Z INFO gateway.monitor received health update {"id": "agent2", "ready": false, "conditions": ["plugin_alerting: Config Sync Pending", "plugin_metrics: Config Sync Pending"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream received notify event for 'alerting/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream processing discovery complete event + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream received notify event for 'metrics/pkg/agent' {"type": "DiscoveryComplete"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream processing discovery complete event + 2023-08-15T01:41:01Z INFO agent gateway connected + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream stream client is now available + 2023-08-15T01:41:01Z DEBUG plugin.alerting/pkg/agent.stream stream server started + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream stream client is now available + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/agent.stream stream server started + 2023-08-15T01:41:01Z DEBUG plugin.metrics syncing metrics node + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics.metrics-backend synced node {"id": "agent2", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting updating node config {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z INFO plugin.metrics condition cleared {"condition": "Config Sync", "previous": "Pending"} + 2023-08-15T01:41:01Z INFO plugin.metrics updating metrics node config + 2023-08-15T01:41:01Z INFO gateway.monitor received health update {"id": "agent2", "ready": true, "conditions": []} + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.metrics metrics capability config updated {"nodeId": "agent2"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics rule sync is disabled + 2023-08-15T01:41:01Z DEBUG test.env configuring node {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:41:01Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "prometheus"} + 2023-08-15T01:41:01Z DEBUG test.env configuring node {"node": "agent2", "driver": "otel"} + 2023-08-15T01:41:01Z DEBUG test.env nothing to do (already stopped) {"node": "agent2", "driver": "otel"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting alerting capability updated {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting rule sync is disabled {"component": "rule-streamer", "nodeId": "agent2"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z DEBUG plugin.metrics sent node health update + 2023-08-15T01:41:01Z DEBUG plugin.alerting send node health update {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "(all)", "capability": "alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics/pkg/gateway.stream stream server started + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} +• [0.063 seconds] +------------------------------ +Node Config when setting a config for a node should return the new config for that node [integration] +/home/ec2-user/opni/test/plugins/alerting/node_config_test.go:175 + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent1", "capability": "alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} +• [0.124 seconds] +------------------------------ +Node Config when resetting a config for a node should return the default config for that node [integration] +/home/ec2-user/opni/test/plugins/alerting/node_config_test.go:206 + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent1", "capability": "alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} +• [0.127 seconds] +------------------------------ +Node Config when resetting the default config should return the original default config for all nodes [integration] +/home/ec2-user/opni/test/plugins/alerting/node_config_test.go:231 + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "(all)", "capability": "alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} +• [0.030 seconds] +------------------------------ +Node Config when setting a config for a node that is the same as the default should preserve the config for that node if the default changes [integration] +/home/ec2-user/opni/test/plugins/alerting/node_config_test.go:250 + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "agent1", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "agent1", "capability": "alerting"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:01Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:01Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:01Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z DEBUG gateway.sync sending sync request to agent {"agentId": "", "capabilities": ["alerting"]} + 2023-08-15T01:41:01Z DEBUG plugin.metrics ignoring sync request due to capability filter + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent1", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting received alerting node sync request {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting syncing alerting node {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z INFO plugin.alerting node sync requested {"component": "node-backend", "cluster": "(all)", "capability": "alerting"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting synced node {"component": "node-backend", "id": "agent2", "time": "2023-08-15T01:41:01Z"} + 2023-08-15T01:41:01Z INFO plugin.alerting alerting node config is up to date {"component": "health-cfg-sync"} + 2023-08-15T01:41:01Z DEBUG plugin.alerting closing cortex cluster status watcher... {"watcher": "cortex-cluster-status"} + 2023-08-15T01:41:01Z INFO plugin.alerting shutting down cluster driver update handler + 2023-08-15T01:41:01Z INFO plugin.alerting exiting main sync loop + 2023-08-15T01:41:01Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:01Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error + 2023-08-15T01:41:01Z WARN plugin.alerting failed to receive cluster health status from grpc stream, retrying... + 2023-08-15T01:41:01Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:01Z INFO gateway shutting down plugins + 2023-08-15T01:41:01Z DEBUG gateway.sync agent disconnected {"id": "agent1"} + 2023-08-15T01:41:01Z DEBUG gateway.delegate agent disconnected {"id": "agent1"} + 2023-08-15T01:41:01Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:01Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:41:01Z INFO gateway all plugins shut down + 2023-08-15T01:41:01Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:01Z ERROR plugin.alerting failed to receive cluster event : rpc error: code = Unavailable desc = error reading from server: read unix @->/tmp/plugin308284646: use of closed network connection + 2023-08-15T01:41:01Z WARN plugin.alerting/pkg/agent.stream stream disconnected with error + 2023-08-15T01:41:01Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error + 2023-08-15T01:41:01Z ERROR plugin.alerting EOF + 2023-08-15T01:41:01Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error + 2023-08-15T01:41:01Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:01Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:41:01Z WARN plugin.alerting/pkg/alerting.stream stream server exited with error + 2023-08-15T01:41:01Z DEBUG gateway.delegate agent disconnected {"id": "agent2"} + 2023-08-15T01:41:01Z DEBUG gateway.sync agent disconnected {"id": "agent2"} + 2023-08-15T01:41:01Z INFO gateway.grpc agent stream closing {"error": "context canceled"} + 2023-08-15T01:41:01Z WARN plugin.metrics/pkg/gateway.stream stream server exited with error {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:01Z WARN gateway http server exited with error {"error": "context canceled"} + 2023-08-15T01:41:01Z WARN agent disconnected from gateway {"error": "rpc error: code = Canceled desc = context canceled"} + 2023-08-15T01:41:01Z WARN agent shutting down gateway client {"error": "context canceled"} + 2023-08-15T01:41:01Z WARN test.env gateway server exited with error {"error": "context canceled"} + 2023-08-15T01:41:01Z ERROR test agent "agent2" exited: context canceled + 2023-08-15T01:41:01Z WARN plugin.metrics/pkg/agent.stream stream disconnected with error + 2023-08-15T01:41:01Z ERROR test agent "agent1" exited: context canceled + 2023-08-15T01:41:01Z WARN test.env management server exited with error {"error": "context canceled"} +• [0.157 seconds] +------------------------------ +Alerting Cluster Integration tests when Installing the Alerting Cluster should install the alerting cluster [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:160 + 2023-08-15T01:41:01Z DEBUG plugin.alerting-test-cluster-driver Syncer start : alerting-server --syncer.alertmanager.config.file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml --syncer.listen.address=:43299 --syncer.alertmanager.address=http://127.0.0.1:39527 --syncer.gateway.join.address=:42179 syncer {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:01Z INFO plugin.alerting-test-cluster-driver Starting AlertManager {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "alertmanager-port": 39527, "opni-port": 43851} + ts=2023-08-15T01:41:01.848Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:41:01.848Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:41:01.851Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=43159 + ts=2023-08-15T01:41:01.852Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + ts=2023-08-15T01:41:01.885Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:01.885Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:01.888Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:39527 + ts=2023-08-15T01:41:01.888Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:39527 + 2023-08-15T01:41:02Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:02Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:02Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:02Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:02Z INFO plugin.alerting-test-cluster-driver Starting AlertManager Syncer {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "port": 43299} + 2023-08-15T01:41:02Z INFO plugin.alerting-test-cluster-driver AlertManager started {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "address": "http://127.0.0.1:39527"} + 2023-08-15T01:41:02Z INFO plugin.alerting updating known peers : [{http://127.0.0.1:39527 http://127.0.0.1:43869}] +• [1.020 seconds] +------------------------------ +Alerting Cluster Integration tests when Installing the Alerting Cluster should apply the configuration [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:176 + 2023-08-15T01:41:02Z DEBUG plugin.alerting-test-cluster-driver Syncer start : alerting-server --syncer.alertmanager.config.file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml --syncer.listen.address=:34301 --syncer.alertmanager.address=http://127.0.0.1:35817 --syncer.gateway.join.address=:42179 syncer {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:02Z INFO plugin.alerting-test-cluster-driver Starting AlertManager {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "alertmanager-port": 35817, "opni-port": 45277} + 2023-08-15T01:41:02Z DEBUG commands/alerting.go:72 syncer gateway join address:42179 + 2023-08-15T01:41:02Z INFO plugin.alerting-syncer syncer/syncer_server.go:90 starting alerting syncer server as identity 347a9035-d7e3-4fd9-82da-706a094b325b + ts=2023-08-15T01:41:02.863Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:41:02.863Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + ts=2023-08-15T01:41:02.866Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=35365 + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:108 acquired gateway alerting client + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:144 starting alerting syncer stream subscription + 2023-08-15T01:41:02Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "a2d53738-539c-49d0-9e03-5398bbd31505"} + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:150 starting new stream subscription + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + ts=2023-08-15T01:41:02.869Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + 2023-08-15T01:41:02Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "461299ca-49e7-4b9f-9be4-4294f13cf095"} + 2023-08-15T01:41:02Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:02.883Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:02.884Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:02Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:02.904Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:02.905Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:02.907Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:35817 + ts=2023-08-15T01:41:02.907Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:35817 + 2023-08-15T01:41:03Z INFO plugin.alerting-test-cluster-driver Starting AlertManager Syncer {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "port": 34301} + 2023-08-15T01:41:03Z INFO plugin.alerting-test-cluster-driver AlertManager started {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "address": "http://127.0.0.1:35817"} + 2023-08-15T01:41:03Z DEBUG plugin.alerting-test-cluster-driver Syncer start : alerting-server --syncer.alertmanager.config.file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml --syncer.listen.address=:46791 --syncer.alertmanager.address=http://127.0.0.1:36617 --syncer.gateway.join.address=:42179 syncer {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:03Z INFO plugin.alerting-test-cluster-driver Starting AlertManager {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "alertmanager-port": 36617, "opni-port": 41133} + ts=2023-08-15T01:41:03.853Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=2 elapsed=2.001124666s + 2023-08-15T01:41:03Z DEBUG commands/alerting.go:72 syncer gateway join address:42179 + 2023-08-15T01:41:03Z INFO plugin.alerting-syncer syncer/syncer_server.go:90 starting alerting syncer server as identity 718140ab-0c49-494d-907b-d6f92289873c + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:108 acquired gateway alerting client + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:144 starting alerting syncer stream subscription + 2023-08-15T01:41:03Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "1b66a932-21d1-4ce6-a23a-a823c793ec18"} + ts=2023-08-15T01:41:03.956Z caller=alertmanager_main.go:259 level=info msg="Starting Alertmanager" version="(version=, branch=, revision=64e8ad84fa6fe91fb5e3aa2828e2240b7169414e)" + ts=2023-08-15T01:41:03.957Z caller=alertmanager_main.go:260 level=info build_context="(go=go1.21.0, platform=linux/amd64, user=, date=, tags=nomsgpack)" + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:150 starting new stream subscription + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:41:03Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "eac5b487-db34-44da-8c98-ed6fea524cc5"} + ts=2023-08-15T01:41:03.965Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.31.7.66 port=41425 + ts=2023-08-15T01:41:03.968Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s + 2023-08-15T01:41:03Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:03.972Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:03.972Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:03Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.004Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.005Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.007Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:36617 + ts=2023-08-15T01:41:04.007Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:36617 + 2023-08-15T01:41:04Z INFO plugin.alerting-test-cluster-driver Starting AlertManager Syncer {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "port": 46791} + 2023-08-15T01:41:04Z INFO plugin.alerting-test-cluster-driver AlertManager started {"config-file": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml", "address": "http://127.0.0.1:36617"} + 2023-08-15T01:41:04Z INFO plugin.alerting updating known peers : [{http://127.0.0.1:39527 http://127.0.0.1:43869} {http://127.0.0.1:35817 http://127.0.0.1:43869} {http://127.0.0.1:36617 http://127.0.0.1:43869}] + 2023-08-15T01:41:04Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:04Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:04Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} +• [2.030 seconds] +------------------------------ +Alerting Cluster Integration tests when Installing the Alerting Cluster the alerting plugin components should be running and healthy [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:204 + 2023-08-15T01:41:04Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:04Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:04Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:04Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:04Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:04Z DEBUG api => {"method": "GET", "path": "/plugin_alerting/ready", "to": "127.0.0.1:35153 (plugin_alerting)", "for": "127.0.0.1", "host": "localhost:37611", "scheme": "https"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:04Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:04Z DEBUG api => {"method": "GET", "path": "/plugin_alerting/healthy", "to": "127.0.0.1:35153 (plugin_alerting)", "for": "127.0.0.1", "host": "localhost:37611", "scheme": "https"} +• [0.036 seconds] +------------------------------ +Alerting Cluster Integration tests when Installing the Alerting Cluster should be able to create some endpoints [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:239 + STEP: verifying they are externally persisted @ 08/15/23 01:41:04.85 + STEP: verifying they are reachable @ 08/15/23 01:41:04.857 + 2023-08-15T01:41:04Z DEBUG plugin.alerting sent manual sync request {"method": "sendManualSyncRequest", "sync-id": "de3890376c48dd11c94cefabc6a269ffaed3e7418bc3eef8bcc0b015d661aca1"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (de3890376c48dd11c94cefabc6a269ffaed3e7418bc3eef8bcc0b015d661aca1) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received request to trigger alerts on condition id:"eCX4oAdVdZHpaC5g7h4xUX" {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received alert annotations : map[OpniHeader:Test notification OpniSummary:Admin has sent a test notification] {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (de3890376c48dd11c94cefabc6a269ffaed3e7418bc3eef8bcc0b015d661aca1) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.870Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=3 elapsed=2.000885063s + ts=2023-08-15T01:41:04.870Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.871Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.871Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.871Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting sent manual sync request {"method": "sendManualSyncRequest", "sync-id": "f15b556b3ab0d0701569f65cbf4f08f871a4b9ddf73916e3f54c6d65b276a387"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received request to trigger alerts on condition id:"7kFfA7bgQ9ukcD3Zqo7h7Q" {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received alert annotations : map[OpniHeader:Test notification OpniSummary:Admin has sent a test notification] {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (f15b556b3ab0d0701569f65cbf4f08f871a4b9ddf73916e3f54c6d65b276a387) config message + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (f15b556b3ab0d0701569f65cbf4f08f871a4b9ddf73916e3f54c6d65b276a387) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.884Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.884Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.885Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.885Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG commands/alerting.go:72 syncer gateway join address:42179 + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:90 starting alerting syncer server as identity 4ea3286b-3229-421f-9efb-e5a702a331a8 + 2023-08-15T01:41:04Z DEBUG plugin.alerting sent manual sync request {"method": "sendManualSyncRequest", "sync-id": "2423c2d3586fb0857863f8e21bcf39c3c60b98341f8ba81189b64d3031b79c6d"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received request to trigger alerts on condition id:"SGuVCs9FnNm3UaHr6Bm6Y" {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received alert annotations : map[OpniHeader:Test notification OpniSummary:Admin has sent a test notification] {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (2423c2d3586fb0857863f8e21bcf39c3c60b98341f8ba81189b64d3031b79c6d) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (2423c2d3586fb0857863f8e21bcf39c3c60b98341f8ba81189b64d3031b79c6d) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.898Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.899Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.900Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.899Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:108 acquired gateway alerting client + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:144 starting alerting syncer stream subscription + 2023-08-15T01:41:04Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "91dd5c2f-3786-4562-870a-0f052b40ecbd"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:150 starting new stream subscription + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:124 trying to acquire remote syncer stream + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:133 connected to remote syncer + 2023-08-15T01:41:04Z DEBUG plugin.alerting sent manual sync request {"method": "sendManualSyncRequest", "sync-id": "4775d46c83afd9b12874d7602b38590d5db61b43c982aab4018b64be7fbe4859"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received request to trigger alerts on condition id:"QMH3opSfy9EGrSBnyp56X4" {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received alert annotations : map[OpniHeader:Test notification OpniSummary:Admin has sent a test notification] {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z INFO plugin.alerting remote syncer connected, performaing initial sync... {"method": "SyncConfig", "assignedId": "274ca105-9abb-4f72-a7e7-53dedf1980f4"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (4775d46c83afd9b12874d7602b38590d5db61b43c982aab4018b64be7fbe4859) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (4775d46c83afd9b12874d7602b38590d5db61b43c982aab4018b64be7fbe4859) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.909Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.909Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.909Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.909Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (127fe9181331fbe2847a95a2797367ef9dd375c862ac9b44cf9df5f5c023a3a6) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.924Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.925Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting sent manual sync request {"method": "sendManualSyncRequest", "sync-id": "127fe9181331fbe2847a95a2797367ef9dd375c862ac9b44cf9df5f5c023a3a6"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received request to trigger alerts on condition id:"oc2ac5Wcs6uAzgERrTsdvc" {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting Received alert annotations : map[OpniHeader:Test notification OpniSummary:Admin has sent a test notification] {"component": "notifications", "Handler": "TriggerAlerts"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (127fe9181331fbe2847a95a2797367ef9dd375c862ac9b44cf9df5f5c023a3a6) config message + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (127fe9181331fbe2847a95a2797367ef9dd375c862ac9b44cf9df5f5c023a3a6) config message + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:04.933Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.934Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.933Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:04.934Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (127fe9181331fbe2847a95a2797367ef9dd375c862ac9b44cf9df5f5c023a3a6) config message + 2023-08-15T01:41:04Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:04Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:05Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:05Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:05Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:05Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:05Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:05Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:05Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:05Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:05Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:05Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:05Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:220 put config request received {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:05.631Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:05.631Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:05.631Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:05.631Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:05.632Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + ts=2023-08-15T01:41:05.632Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml + 2023-08-15T01:41:05Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:05Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + 2023-08-15T01:41:05Z DEBUG plugin.alerting-syncer syncer/syncer_server.go:246 config reloaded {"config-path": "/tmp/opni-test-2132242568/alertmanager-config-5dc62f01-96b3-4301-9b89-d83632b4c5e7/alertmanager.yaml"} + ts=2023-08-15T01:41:05.854Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=1 before=2 now=3 elapsed=4.002109339s + 2023-08-15T01:41:05Z INFO plugin.alerting evaluation context is exiting, exiting evaluation loop {"component": "alarms", "onSystemConditionCreate": "34e6zCJWGxLSxNSP97cftd"} + 2023-08-15T01:41:05Z INFO plugin.alerting parent context is exiting, exiting evaluation loop {"component": "alarms", "onCapabilityStatusCreate": "stdRzQww4KPXDy5r3dVAh6"} + ts=2023-08-15T01:41:05.969Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=3 elapsed=2.000338425s + 2023-08-15T01:41:06Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:06Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:06Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:06Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:06Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:06Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:06Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:06Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:06Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:06Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:06Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:07Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:07Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:07Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:07Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:07Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:07Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:07Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:07Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:07Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:07Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:07Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:08Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:08Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:08Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:08Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:08Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:08Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:08Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:08Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:08Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:08Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:08Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:09Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:09Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:09Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:09Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:09Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:09Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:09Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:09Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:09Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:09Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:09Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:10Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:10Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:10Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:10Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:10Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:10Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:10Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:10Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:10Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:10Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:10Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:10Z INFO plugin.alerting successfully synced (1/5) conditions {"component": "alarms"} + 2023-08-15T01:41:10Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 3/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:10Z ERROR plugin.alerting context canceled + nats: timeout {"action": "runSyncTasks"} + 2023-08-15T01:41:10Z ERROR plugin.alerting failed to successfully run all alerting sync tasks : context canceled + nats: timeout + 2023-08-15T01:41:10Z ERROR plugin.alerting skipping alerting periodic sync due to error : %scontext canceled {"action": "runSyncTasks"} + 2023-08-15T01:41:10Z ERROR plugin.alerting failed to successfully run all alerting sync tasks : context canceled + 2023-08-15T01:41:10Z INFO plugin.alerting exiting main sync loop + 2023-08-15T01:41:11Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:11Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:11Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:11Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:11Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:11Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:11Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:11Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:11Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:11Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:11Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:12Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:12Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:12Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:12Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:12Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:12Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:12Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:12Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:12Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:12Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:12Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + ts=2023-08-15T01:41:12.873Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004204209s + 2023-08-15T01:41:13Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:13Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:13Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:13Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:13Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:13Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:13Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:13Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:13Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:13Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:13Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + ts=2023-08-15T01:41:13.856Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=12.00388903s + ts=2023-08-15T01:41:13.857Z caller=notify.go:740 level=warn component=dispatcher receiver=opni.default.hook integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 127.0.0.1:3000: connect: connection refused" + ts=2023-08-15T01:41:13.971Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.00257536s + 2023-08-15T01:41:14Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:14Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:14Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:14Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:14Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:14Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:14Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:14Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:14Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:14Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:14Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:15Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:15Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:15Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:15Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:15Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:15Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:15Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:15Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:15Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:15Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:15Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:16Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:16Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:16Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:16Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:16Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:16Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:16Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:16Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:16Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:16Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:16Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:17Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:17Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:17Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:17Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:17Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:17Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:17Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:17Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:17Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:17Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:17Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:18Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:18Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:18Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:18Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:18Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:18Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:18Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:18Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:18Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:18Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:18Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:19Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:19Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:19Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:19Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:19Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:19Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:19Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:19Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:19Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:19Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:19Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:20Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:20Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:20Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:20Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:20Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:20Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:20Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:20Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:20Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:20Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:20Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:21Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:21Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:21Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:21Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:21Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:21Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:21Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:21Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:21Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:21Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:21Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:22Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:22Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:22Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:22Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:22Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:22Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:22Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:22Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:22Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:22Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:22Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:23Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:23Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:23Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:23Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:23Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:23Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:23Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:23Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:23Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:23Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:23Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:24Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:24Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:24Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:24Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:24Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:24Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:24Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:24Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:24Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:24Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:24Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:25Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:25Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:25Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:25Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:25Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:25Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:25Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:25Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:25Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:25Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:25Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:26Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:26Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:26Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:26Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:26Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:26Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:26Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:26Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:26Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:26Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:26Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:27Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:27Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:27Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:27Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:27Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:27Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:27Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:27Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:27Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:27Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:27Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:28Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:28Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:28Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:28Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:28Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:28Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:28Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:28Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:28Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:28Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:28Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:29Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:29Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:29Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:29Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:29Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:29Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:29Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:29Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:29Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:29Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:29Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:30Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:30Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:30Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:30Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:30Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:30Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:30Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:30Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:30Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:30Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:30Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:31Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:31Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:31Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:31Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:31Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:31Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:31Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:31Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:31Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:31Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:31Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:32Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:32Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:32Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:32Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:32Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:32Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:32Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:32Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:32Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:32Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:32Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:33Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:33Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:33Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:33Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:33Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:33Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:34Z INFO plugin.alerting Running periodic sync for alerting {"action": "runSyncTasks"} + 2023-08-15T01:41:34Z INFO plugin.alerting syncing 0 conditions {"component": "alarms"} + 2023-08-15T01:41:34Z INFO plugin.alerting successfully synced (0/0) conditions {"component": "alarms"} + 2023-08-15T01:41:34Z DEBUG plugin.alerting sync change detected, pushing sync request to remote syncers {"method": "doSync", "syncId": "e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150"} + 2023-08-15T01:41:34Z INFO plugin.alerting finished running periodic sync for alerting, sucessfully ran 4/4 sync tasks {"action": "runSyncTasks"} + 2023-08-15T01:41:34Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:34Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:34Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:34Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + 2023-08-15T01:41:34Z INFO plugin.alerting-syncer syncer/syncer_server.go:173 received sync (e0849115578d79a76d525e44b6d6fa06b3acc3370cd54602f84401f75da13150) config message + 2023-08-15T01:41:34Z INFO plugin.alerting-syncer syncer/syncer_server.go:175 already up to date + [FAILED] in [It] - /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:269 @ 08/15/23 01:41:34.934 + [FAILED] in [AfterEach] - /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:153 @ 08/15/23 01:41:34.944 +• [FAILED] [30.109 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should be able to create some endpoints [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:239 + + [FAILED] Timed out after 30.001s. + Expected success, but got an error: + <*errors.errorString | 0xc0054da220>: + server mock-integration-fAeNNrkhJqtd26oSstsDta did not receive any alerts + { + s: "server mock-integration-fAeNNrkhJqtd26oSstsDta did not receive any alerts", + } + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:269 @ 08/15/23 01:41:34.934 + + There were additional failures detected. To view them in detail run ginkgo -vv +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should create some default conditions when bootstrapping agents [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:277 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:277 @ 08/15/23 01:41:34.945 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] shoud list conditions by given filters [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:335 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:335 @ 08/15/23 01:41:34.946 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should be able to attach endpoints to conditions [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:401 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:401 @ 08/15/23 01:41:34.947 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] agent disconnect alarms should fire when agents are disconnected  [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:505 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:505 @ 08/15/23 01:41:34.948 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should be able to batch list status and filter by status [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:601 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:601 @ 08/15/23 01:41:34.948 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should be able to push notifications to our notification endpoints [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:627 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:627 @ 08/15/23 01:41:34.949 +------------------------------ +P [PENDING] +Alerting Cluster Integration tests when Installing the Alerting Cluster should be able to list opni messages [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:651 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should return warnings when trying to edit/delete alert endpoints that are involved in conditions [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:671 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:671 @ 08/15/23 01:41:34.95 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should have a functional timeline [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:706 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:706 @ 08/15/23 01:41:34.951 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] the alertmanager proxy served by the Gateway HTTP port should be able to list the alarms [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:757 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:757 @ 08/15/23 01:41:34.952 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should sync friendly cluster names to conditions [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:782 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:782 @ 08/15/23 01:41:34.953 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should force update/delete alert endpoints involved in conditions [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:818 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:818 @ 08/15/23 01:41:34.953 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should delete the downstream agents [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:881 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:881 @ 08/15/23 01:41:34.954 +------------------------------ +S [SKIPPED] [0.000 seconds] +Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should uninstall the alerting cluster [integration] +/home/ec2-user/opni/test/plugins/alerting/alerting_test.go:891 + + [SKIPPED] Spec skipped because an earlier spec in an ordered container failed + In [It] at: /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:891 @ 08/15/23 01:41:34.955 +------------------------------ +[DeferCleanup (Suite)]  +/home/ec2-user/opni/test/plugins/alerting/alerting_suite_test.go:51 + 2023-08-15T01:41:34Z DEBUG plugin.alerting closing cortex cluster status watcher... {"watcher": "cortex-cluster-status"} + 2023-08-15T01:41:34Z INFO plugin.alerting exiting main sync loop + 2023-08-15T01:41:34Z INFO plugin.alerting shutting down cluster driver update handler + 2023-08-15T01:41:34Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "461299ca-49e7-4b9f-9be4-4294f13cf095"} + 2023-08-15T01:41:34Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:34Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "a2d53738-539c-49d0-9e03-5398bbd31505"} + 2023-08-15T01:41:34Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "91dd5c2f-3786-4562-870a-0f052b40ecbd"} + 2023-08-15T01:41:34Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "1b66a932-21d1-4ce6-a23a-a823c793ec18"} + 2023-08-15T01:41:34Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "eac5b487-db34-44da-8c98-ed6fea524cc5"} + 2023-08-15T01:41:34Z DEBUG plugin.alerting exiting syncer loop, alerting plugin shutting down {"method": "SyncConfig", "assignedId": "274ca105-9abb-4f72-a7e7-53dedf1980f4"} + 2023-08-15T01:41:34Z WARN plugin.alerting failed to receive cluster health status from grpc stream, retrying... + ts=2023-08-15T01:41:34.957Z caller=alertmanager_main.go:601 level=info msg="Received SIGTERM, exiting gracefully..." + 2023-08-15T01:41:34Z ERROR plugin.alerting failed to receive cluster event : rpc error: code = Canceled desc = context canceled + ts=2023-08-15T01:41:34.957Z caller=alertmanager_main.go:601 level=info msg="Received SIGTERM, exiting gracefully..." + 2023-08-15T01:41:34Z INFO gateway shutting down plugins + 2023-08-15T01:41:34Z WARN test.env management server exited with error {"error": "context canceled"} + 2023-08-15T01:41:34Z WARN gateway http server exited with error {"error": "context canceled"} + 2023-08-15T01:41:34Z WARN test.env gateway server exited with error {"error": "context canceled"} + 2023-08-15T01:41:34Z INFO gateway all plugins shut down + 2023-08-15T01:41:34Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:34Z WARN jetstream disconnected from jetstream + 2023-08-15T01:41:34Z ERROR plugin.alerting EOF +[DeferCleanup (Suite)] PASSED [0.024 seconds] +------------------------------ +[DeferCleanup (Suite)]  +/home/ec2-user/go/pkg/mod/github.com/golang/mock@v1.7.0-rc.1/gomock/controller.go:99 +[DeferCleanup (Suite)] PASSED [0.000 seconds] +------------------------------ + +Summarizing 1 Failure: + [FAIL] Alerting Cluster Integration tests when Installing the Alerting Cluster [It] should be able to create some endpoints [integration] + /home/ec2-user/opni/test/plugins/alerting/alerting_test.go:269 + +Ran 27 of 41 Specs in 99.430 seconds +FAIL! -- 26 Passed | 1 Failed | 1 Pending | 13 Skipped +--- FAIL: TestAlerting (99.43s) +FAIL + +Ginkgo ran 1 suite in 1m49.83826659s + +Test Suite Failed diff --git a/test/plugins/metrics/cortex_query_test.go b/test/plugins/metrics/cortex_query_test.go index f5b71aa3b2..56176adb04 100644 --- a/test/plugins/metrics/cortex_query_test.go +++ b/test/plugins/metrics/cortex_query_test.go @@ -119,5 +119,7 @@ var _ = Describe("Cortex query tests", Ordered, Label("integration"), func() { body, err := io.ReadAll(resp.Body) Expect(err).NotTo(HaveOccurred()) Expect(body).NotTo(BeEmpty()) + + // TODO : MAYBE add other cortex APIS that query metric data in special ways }) }) diff --git a/test/plugins/metrics/cortexadmin_test.go b/test/plugins/metrics/cortexadmin_test.go deleted file mode 100644 index eafee3f8a9..0000000000 --- a/test/plugins/metrics/cortexadmin_test.go +++ /dev/null @@ -1,490 +0,0 @@ -package metrics_test - -import ( - "context" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "time" - - "github.com/rancher/opni/pkg/alerting/metrics/naming" - "github.com/rancher/opni/pkg/capabilities/wellknown" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - "github.com/rancher/opni/plugins/metrics/apis/cortexops" - "github.com/tidwall/gjson" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - capabilityv1 "github.com/rancher/opni/pkg/apis/capability/v1" - corev1 "github.com/rancher/opni/pkg/apis/core/v1" - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - storagev1 "github.com/rancher/opni/pkg/apis/storage/v1" - "github.com/rancher/opni/pkg/test" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/durationpb" - "google.golang.org/protobuf/types/known/emptypb" -) - -type TestSeriesMetrics struct { - input *cortexadmin.SeriesRequest - output *cortexadmin.SeriesInfoList -} - -type TestMetricLabelSet struct { - input *cortexadmin.LabelRequest - output *cortexadmin.MetricLabels -} - -func expectRuleGroupToExist(ctx context.Context, adminClient cortexadmin.CortexAdminClient, tenant string, groupName string, expectedYaml []byte) error { - for i := 0; i < 10; i++ { - resp, err := adminClient.GetRule(ctx, &cortexadmin.GetRuleRequest{ - ClusterId: tenant, - Namespace: "test", - GroupName: groupName, - }) - if err == nil { - Expect(resp.Data).To(Not(BeNil())) - Expect(resp.Data).To(MatchYAML(expectedYaml)) - return nil - } - time.Sleep(1) - } - return fmt.Errorf("Rule %s should exist, but doesn't", groupName) -} - -func expectRuleGroupToNotExist(ctx context.Context, adminClient cortexadmin.CortexAdminClient, tenant string, groupName string) error { - for i := 0; i < 10; i++ { - _, err := adminClient.GetRule(ctx, &cortexadmin.GetRuleRequest{ - ClusterId: tenant, - Namespace: "test", - GroupName: groupName, - }) - if err != nil { - Expect(status.Code(err)).To(Equal(codes.NotFound)) - return nil - } - - time.Sleep(1) - } - return fmt.Errorf("Rule %s still exists, but shouldn't", groupName) -} - -type mockPod struct { - podName string - namespace string - phase string - uid string -} - -func setMockKubernetesPodState(kubePort int, pod *mockPod) { - queryUrl := fmt.Sprintf("http://localhost:%d/set", kubePort) - client := &http.Client{ - Transport: &http.Transport{}, - } - req, err := http.NewRequest("GET", queryUrl, nil) - if err != nil { - panic(err) - } - values := url.Values{} - values.Set("obj", "pod") - values.Set("name", pod.podName) - values.Set("namespace", pod.namespace) - values.Set("phase", pod.phase) - values.Set("uid", pod.uid) - req.URL.RawQuery = values.Encode() - go func() { - resp, err := client.Do(req) - if err != nil { - panic(err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - panic(fmt.Sprintf("kube metrics prometheus collector hit an error %d", resp.StatusCode)) - } - }() -} - -var _ = Describe("Converting ServiceLevelObjective Messages to Prometheus Rules", Ordered, Label("integration", "slow"), func() { - ctx := context.Background() - var env *test.Environment - var adminClient cortexadmin.CortexAdminClient - var kubernetesTempMetricServerPort int - var kubernetesJobName string - ruleTestDataDir := "../../../pkg/test/testdata/testdata/slo/cortexrule" - BeforeAll(func() { - env = &test.Environment{} - Expect(env.Start()).To(Succeed()) - DeferCleanup(env.Stop) - - opsClient := cortexops.NewCortexOpsClient(env.ManagementClientConn()) - _, err := opsClient.ConfigureCluster(context.Background(), &cortexops.ClusterConfiguration{ - Mode: cortexops.DeploymentMode_AllInOne, - Storage: &storagev1.StorageSpec{ - Backend: storagev1.Filesystem, - }, - }) - Expect(err).NotTo(HaveOccurred()) - - client := env.NewManagementClient() - token, err := client.CreateBootstrapToken(context.Background(), &managementv1.CreateBootstrapTokenRequest{ - Ttl: durationpb.New(time.Hour), - }) - Expect(err).NotTo(HaveOccurred()) - info, err := client.CertsInfo(context.Background(), &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - adminClient = cortexadmin.NewCortexAdminClient(env.ManagementClientConn()) - // wait until data has been stored in cortex for the cluster - kubernetesTempMetricServerPort = env.StartMockKubernetesMetricServer() - fmt.Printf("Mock kubernetes metrics server started on port %d\n", kubernetesTempMetricServerPort) - _, errc := env.StartAgent("agent", token, []string{info.Chain[len(info.Chain)-1].Fingerprint}) - Eventually(errc).Should(Receive(BeNil())) - kubernetesJobName = "kubernetes" - env.SetPrometheusNodeConfigOverride("agent", test.NewOverridePrometheusConfig( - "alerting/prometheus/config.yaml", - []test.PrometheusJob{ - { - JobName: kubernetesJobName, - ScrapePort: kubernetesTempMetricServerPort, - }, - }), - ) - _, errc2 := env.StartAgent("agent2", token, []string{info.Chain[len(info.Chain)-1].Fingerprint}) - Eventually(errc2).Should(Receive(BeNil())) - - _, err = client.InstallCapability(context.Background(), &managementv1.CapabilityInstallRequest{ - Name: wellknown.CapabilityMetrics, - Target: &capabilityv1.InstallRequest{ - Cluster: &corev1.Reference{Id: "agent"}, - }, - }) - Expect(err).NotTo(HaveOccurred()) - - _, err = client.InstallCapability(context.Background(), &managementv1.CapabilityInstallRequest{ - Name: wellknown.CapabilityMetrics, - Target: &capabilityv1.InstallRequest{ - Cluster: &corev1.Reference{Id: "agent2"}, - }, - }) - Expect(err).NotTo(HaveOccurred()) - - Eventually(func() error { - stats, err := adminClient.AllUserStats(context.Background(), &emptypb.Empty{}) - if err != nil { - return err - } - for _, item := range stats.Items { - if item.UserID == "agent" { - if item.NumSeries > 0 { - return nil - } - } - } - return fmt.Errorf("waiting for metric data to be stored in cortex") - }, 60*time.Second, 1*time.Second).Should(Succeed()) - - Eventually(func() error { - stats, err := adminClient.AllUserStats(context.Background(), &emptypb.Empty{}) - if err != nil { - return err - } - for _, item := range stats.Items { - if item.UserID == "agent2" { - if item.NumSeries > 0 { - return nil - } - } - } - return fmt.Errorf("waiting for metric data to be stored in cortex") - }, 60*time.Second, 1*time.Second).Should(Succeed()) - - //scrape interval is 1 second - }) - - When("We use the cortex admin plugin", func() { - It("Should be able to enumerate all kube metrics series that match a certain expression", func() { - setMockKubernetesPodState(kubernetesTempMetricServerPort, &mockPod{ - podName: "test-pod", - namespace: "test-namespace", - phase: "Running", - uid: "test-uid", - }) - Eventually(func() error { - resp, err := adminClient.ExtractRawSeries(ctx, &cortexadmin.MatcherRequest{ - Tenant: "agent", - MatchExpr: naming.KubeObjMetricNameMatcher, - }, - ) - if err != nil { - return err - } - - result := gjson.Get(string(resp.Data), "data.result") - if !result.Exists() { - return fmt.Errorf("no result data") - } - if len(result.Array()) == 0 { - return fmt.Errorf("no results") - } - return nil - }, 3*time.Minute, 30*time.Second).Should(Succeed()) - - }) - - It("Should be able to list distinct metrics for each job ", func() { - // expected outputs are a subset of the actual outputs - inputs := []TestSeriesMetrics{ - { - input: &cortexadmin.SeriesRequest{ - Tenant: "agent", - JobId: "prometheus", - }, - output: &cortexadmin.SeriesInfoList{ - Items: []*cortexadmin.SeriesInfo{ - { - SeriesName: "up", - }, - { - SeriesName: "prometheus_http_requests_total", - }, - }, - }, - }, - { - input: &cortexadmin.SeriesRequest{ - Tenant: "agent2", - JobId: "prometheus", - }, - output: &cortexadmin.SeriesInfoList{ - Items: []*cortexadmin.SeriesInfo{ - { - SeriesName: "up", - }, - { - SeriesName: "prometheus_http_requests_total", - }, - }, - }, - }, - } - for _, input := range inputs { - resp, err := adminClient.GetSeriesMetrics(ctx, input.input) - Expect(err).NotTo(HaveOccurred()) - for _, expected := range input.output.Items { - found := false - for _, item := range resp.Items { - if item.SeriesName == expected.SeriesName { - //FIXME: when the metadata API is working also check metadata here - found = true - break - } - } - Expect(found).To(BeTrue()) - } - } - }) - - It("should be able to fetch metric label pairs for each metric", func() { - // expected outputs are a subset of the actual outputs - inputs := []TestMetricLabelSet{ - { - input: &cortexadmin.LabelRequest{ - Tenant: "agent", - JobId: "prometheus", - MetricName: "prometheus_http_requests_total", - }, - output: &cortexadmin.MetricLabels{ - Items: []*cortexadmin.LabelSet{ - { - Name: "code", - Items: []string{ - "200", - }, - }, - { - Name: "handler", - Items: []string{ - "/-/ready", - "/metrics", - }, - }, - }, - }, - }, - { - input: &cortexadmin.LabelRequest{ - Tenant: "agent2", - JobId: "prometheus", - MetricName: "prometheus_http_requests_total", - }, - output: &cortexadmin.MetricLabels{ - Items: []*cortexadmin.LabelSet{ - { - Name: "code", - Items: []string{ - "200", - }, - }, - { - Name: "handler", - Items: []string{ - "/-/ready", - }, - }, - }, - }, - }, - } - for _, input := range inputs { - resp, err := adminClient.GetMetricLabelSets(ctx, input.input) - Expect(err).NotTo(HaveOccurred()) - Expect(resp).NotTo(BeNil()) - for _, expected := range input.output.Items { - found := false - for _, item := range resp.Items { - if item.Name == expected.Name { - found = true - for _, expectedItem := range expected.Items { - foundItem := false - for _, item := range item.Items { - if item == expectedItem { - foundItem = true - break - } - } - Expect(foundItem).To(BeTrue()) - } - break - } - } - Expect(found).To(BeTrue()) - } - } - }) - - It("Should be able to create rules from prometheus yaml", func() { - sampleRule := fmt.Sprintf("%s/sampleRule.yaml", ruleTestDataDir) - sampleRuleYamlString, err := ioutil.ReadFile(sampleRule) - Expect(err).To(Succeed()) - _, err = adminClient.LoadRules(ctx, - &cortexadmin.LoadRuleRequest{ - ClusterId: "agent", - Namespace: "test", - YamlContent: sampleRuleYamlString, - }) - Expect(err).To(Succeed()) - - // Note that sloth by default groups its output into a list of rulefmt.RuleGroup called "groups:" - // While we require the list of rulefmt.RuleGroup to be separated by "---\n" - slothGeneratedGroup := fmt.Sprintf("%s/slothGeneratedGroup.yaml", ruleTestDataDir) - slothGeneratedGroupYamlString, err := ioutil.ReadFile(slothGeneratedGroup) - Expect(err).To(Succeed()) - _, err = adminClient.LoadRules(ctx, - &cortexadmin.LoadRuleRequest{ - ClusterId: "agent", - Namespace: "test", - YamlContent: slothGeneratedGroupYamlString, - }) - Expect(err).To(Succeed()) - Expect(err).NotTo(HaveOccurred()) - - Eventually(func() error { - return expectRuleGroupToExist( - ctx, adminClient, "agent", - "opni-test-slo-rule", sampleRuleYamlString) - }).Should(Succeed()) - - }) - - It("Should be able to update existing rule groups", func() { - sampleRuleUpdate := fmt.Sprintf("%s/sampleRuleUpdate.yaml", ruleTestDataDir) - sampleRuleYamlUpdateString, err := ioutil.ReadFile(sampleRuleUpdate) - Expect(err).To(Succeed()) - _, err = adminClient.LoadRules(ctx, - &cortexadmin.LoadRuleRequest{ - ClusterId: "agent", - Namespace: "test", - YamlContent: sampleRuleYamlUpdateString, - }) - Expect(err).To(Succeed()) - - Eventually(func() error { - return expectRuleGroupToExist( - ctx, adminClient, "agent", - "opni-test-slo-rule", sampleRuleYamlUpdateString) - }).Should(Succeed()) - }) - - It("Should be able to delete existing rule groups", func() { - deleteGroupName := "opni-test-slo-rule" - _, err := adminClient.DeleteRule(ctx, &cortexadmin.DeleteRuleRequest{ - ClusterId: "agent", - Namespace: "test", - GroupName: deleteGroupName, - }) - Expect(err).To(Succeed()) - - // Should find no rule named "opni-test-slo-rule" after deletion - Eventually(func() error { - return expectRuleGroupToNotExist( - ctx, adminClient, "agent", - "opni-test-slo-rule") - }).Should(Succeed()) - }) - }) - When("We are in a multitenant environment", func() { - It("Should be able to apply rules across tenants", func() { - sampleRule := fmt.Sprintf("%s/sampleRule.yaml", ruleTestDataDir) - sampleRuleYamlString, err := ioutil.ReadFile(sampleRule) - Expect(err).To(Succeed()) - _, err = adminClient.LoadRules(ctx, - &cortexadmin.LoadRuleRequest{ - ClusterId: "agent", - Namespace: "test", - YamlContent: sampleRuleYamlString, - }) - Expect(err).To(Succeed()) - _, err = adminClient.LoadRules(ctx, - &cortexadmin.LoadRuleRequest{ - Namespace: "test", - ClusterId: "agent2", - YamlContent: sampleRuleYamlString, - }) - Expect(err).To(Succeed()) - - Eventually(func() error { - return expectRuleGroupToExist( - ctx, adminClient, - "agent", "opni-test-slo-rule", sampleRuleYamlString) - }).Should(Succeed()) - - Eventually(func() error { - return expectRuleGroupToExist( - ctx, adminClient, - "agent2", "opni-test-slo-rule", sampleRuleYamlString, - ) - }).Should(Succeed()) - - deleteGroupName := "opni-test-slo-rule" - _, err = adminClient.DeleteRule(ctx, &cortexadmin.DeleteRuleRequest{ - ClusterId: "agent", - Namespace: "test", - GroupName: deleteGroupName, - }) - Expect(err).To(Succeed()) - - Eventually(func() error { - return expectRuleGroupToExist( - ctx, adminClient, "agent2", - "opni-test-slo-rule", sampleRuleYamlString) - }).Should(Succeed()) - - Eventually(func() error { - return expectRuleGroupToNotExist( - ctx, adminClient, "agent", - "opni-test-slo-rule") - }).Should(Succeed()) - }) - }) -}) diff --git a/test/plugins/metrics/slo_test.go b/test/plugins/metrics/slo_test.go new file mode 100644 index 0000000000..96d6dbfd40 --- /dev/null +++ b/test/plugins/metrics/slo_test.go @@ -0,0 +1,685 @@ +package metrics_test + +import ( + "context" + "fmt" + "net/url" + "time" + + "github.com/golang/snappy" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + prommodel "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/rulefmt" + "github.com/prometheus/prometheus/prompb" + "github.com/prometheus/prometheus/storage/remote" + capabilityv1 "github.com/rancher/opni/pkg/apis/capability/v1" + corev1 "github.com/rancher/opni/pkg/apis/core/v1" + managementv1 "github.com/rancher/opni/pkg/apis/management/v1" + slov1 "github.com/rancher/opni/pkg/apis/slo/v1" + storagev1 "github.com/rancher/opni/pkg/apis/storage/v1" + "github.com/rancher/opni/pkg/capabilities" + "github.com/rancher/opni/pkg/capabilities/wellknown" + "github.com/rancher/opni/pkg/logger" + "github.com/rancher/opni/pkg/metrics/compat" + "github.com/rancher/opni/pkg/slo/backend" + "github.com/rancher/opni/pkg/slo/backend/metrics" + "github.com/rancher/opni/pkg/test" + "github.com/rancher/opni/pkg/util" + "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" + "github.com/rancher/opni/plugins/metrics/apis/cortexops" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" + "gopkg.in/yaml.v3" +) + +func init() { + metrics.DefaultEvaluationInterval = 1 * time.Second +} + +var _ = Describe("Prometheus SLOs", Ordered, Label("integration"), func() { + var env *test.Environment + var fingerprint string + var adminClient cortexadmin.CortexAdminClient + var sloGen *metrics.SLOGeneratorImpl + var sloDatasource backend.SLODatasource + var rw remote.WriteClient + var sloRef *corev1.Reference + BeforeAll(func() { + env = &test.Environment{} + Expect(env.Start()).To(Succeed()) + DeferCleanup(env.Stop) + mgmtClient := env.NewManagementClient() + var err error + sloGen, err = metrics.NewSLOGenerator(*backend.SLODataToStruct(&slov1.SLOData{ + Id: "id1", + SLO: util.ProtoClone(tempSLO), + })) + Expect(err).To(BeNil()) + + certsInfo, err := mgmtClient.CertsInfo(context.Background(), &emptypb.Empty{}) + Expect(err).NotTo(HaveOccurred()) + fingerprint = certsInfo.Chain[len(certsInfo.Chain)-1].Fingerprint + Expect(fingerprint).NotTo(BeEmpty()) + + token, err := mgmtClient.CreateBootstrapToken(context.Background(), &managementv1.CreateBootstrapTokenRequest{ + Ttl: durationpb.New(1 * time.Hour), + }) + Expect(err).NotTo(HaveOccurred()) + + agentPort, errC := env.StartAgent("agent", token, []string{fingerprint}) + Eventually(errC).Should(Receive(BeNil())) + + rw, err = remote.NewWriteClient("agent", &remote.ClientConfig{ + URL: &config.URL{URL: util.Must(url.Parse(fmt.Sprintf("http://127.0.0.1:%d/api/agent/push", agentPort)))}, + Timeout: model.Duration(time.Second * 10), + }) + Expect(err).NotTo(HaveOccurred()) + + opsClient := cortexops.NewCortexOpsClient(env.ManagementClientConn()) + _, err = opsClient.ConfigureCluster(context.Background(), &cortexops.ClusterConfiguration{ + Mode: cortexops.DeploymentMode_AllInOne, + Storage: &storagev1.StorageSpec{ + Backend: storagev1.Filesystem, + }, + }) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() error { + clusters, err := mgmtClient.ListClusters(env.Context(), &managementv1.ListClustersRequest{}) + if err != nil { + return err + } + if len(clusters.GetItems()) == 0 { + return fmt.Errorf("no clusters found") + } + return nil + }).Should(Succeed()) + + resp, err := mgmtClient.InstallCapability(context.Background(), &managementv1.CapabilityInstallRequest{ + Name: "metrics", + Target: &capabilityv1.InstallRequest{ + Cluster: &corev1.Reference{ + Id: "agent", + }, + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.Status).To(Equal(capabilityv1.InstallResponseStatus_Success)) + + lg := logger.NewPluginLogger().Named("metrics") + + adminClient = cortexadmin.NewCortexAdminClient(env.ManagementClientConn()) + metricsProvider := metrics.NewProvider(lg.With("component", "slo")) + metricsProvider.Initialize(adminClient) + metricsStoreProvider := metrics.NewMetricsSLOStoreProvider(lg.With("component", "slo")) + metricsStoreProvider.Initialize(adminClient) + + sloDatasource = backend.NewSLODatasource( + metrics.NewMetricsSLOStore( + metricsStoreProvider, + ), + metrics.NewBackend( + metricsProvider, + ), + func(ctx context.Context, clusterId *corev1.Reference) error { + cluster, err := mgmtClient.GetCluster(ctx, &corev1.Reference{ + Id: clusterId.Id, + }) + if err != nil { + return err + } + if !capabilities.Has(cluster, capabilities.Cluster(wellknown.CapabilityMetrics)) { + return fmt.Errorf("requires metrics capability to be installed on cluster %s", cluster.GetId()) + } + return nil + }, + ) + }) + When("we use the prometheus slo generator", func() { + Specify("cortex should be reachable", func() { + Eventually(func() error { + statsList, err := adminClient.AllUserStats(context.Background(), &emptypb.Empty{}) + if err != nil { + return err + } + if len(statsList.Items) == 0 { + return fmt.Errorf("no user stats") + } + if statsList.Items[0].NumSeries == 0 { + return fmt.Errorf("no series") + } + return nil + }, 60*time.Second, 100*time.Millisecond).Should(Succeed()) + }) + It("should construct loadable SLI info rule", func() { + expectLoadRules(env.Context(), "info", adminClient, sloGen.Info()) + }) + + It("should construct loadable SLI period rule ", func() { + expectLoadRules(env.Context(), "period", adminClient, sloGen.Period()) + }) + + It("should construct loadable SLI error budget rule", func() { + expectLoadRules(env.Context(), "errbudget", adminClient, sloGen.ErrorBudget()) + }) + + It("should construct loadable SLI objective rule", func() { + expectLoadRules(env.Context(), "objective", adminClient, sloGen.Objective()) + }) + + It("should construct loadable SLI SLI rule", func() { + expectLoadRules(env.Context(), "objective", adminClient, sloGen.SLI( + metrics.WindowMetadata{ + WindowDur: prommodel.Duration(time.Second * 5), + Name: "test", + })) + }) + + It("should construct SLI current burn rate rule", func() { + expectLoadRules(env.Context(), "currentBurnRate", adminClient, sloGen.CurrentBurnRate()) + }) + + It("should construct SLI period burn rate rule", func() { + expectLoadRules(env.Context(), "periodBurnRate", adminClient, sloGen.PeriodBurnRate()) + }) + + It("should construct error budget remaining rule", func() { + expectLoadRules(env.Context(), "errorBudgetRemaining", adminClient, sloGen.ErrorBudgetRemaining()) + }) + }) + + When("we use the SLO monitoring datasource service discovery backend", func() { + It("should remote write SLO metric data to cortex", func() { + now := time.Now().UnixMilli() + err := remoteWrite(env.Context(), rw, &prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "testmetric"}, + {Name: "job", Value: "testservice"}, + {Name: "a", Value: "a"}, + }, + Samples: []prompb.Sample{{Value: 1, Timestamp: now}}, + }, + }, + }) + Expect(err).To(Succeed()) + }) + + It("should list available services", func() { + Eventually(func() error { + svcList, err := sloDatasource.ListServices(env.Context(), &slov1.ListServicesRequest{ + Datasource: "monitoring", + ClusterId: "agent", + }) + if err != nil { + return err + } + if len(svcList.Items) == 0 { + return fmt.Errorf("no services") + } + if !svcList.ContainsId("prometheus") { + return fmt.Errorf("missing prometheus") + } + if !svcList.ContainsId("testservice") { + return fmt.Errorf("missing testservice") + } + return nil + }).Should(Succeed()) + }) + + It("should list available metrics on a service", func() { + Eventually(func() error { + metricList, err := sloDatasource.ListMetrics(env.Context(), &slov1.ListMetricsRequest{ + Datasource: "monitoring", + ClusterId: "agent", + ServiceId: "testservice", + }) + if err != nil { + return err + } + + if len(metricList.GroupNameToMetrics) == 0 { + return fmt.Errorf("no metrics") + } + + if !metricList.ContainsId("testmetric") { + return fmt.Errorf("missing testmetric") + } + return nil + }).Should(Succeed()) + + }) + + It("should list available events on a (metrics,service) pair", func() { + Eventually(func() error { + eventList, err := sloDatasource.ListEvents(env.Context(), &slov1.ListEventsRequest{ + Datasource: "monitoring", + ClusterId: "agent", + ServiceId: "testservice", + MetricId: "testmetric", + }) + if err != nil { + return err + } + if len(eventList.Items) == 0 { + return fmt.Errorf("no events") + } + if !eventList.ContainsId("a") { + return fmt.Errorf("missing event a") + } + return nil + }).Should(Succeed()) + + }) + }) + + When("we create SLO data", func() { + It("should write SLO data", func() { + dur := prommodel.Duration(time.Hour * 2) + err := writeSLO( + env.Context(), + rw, + time.Duration(dur), + "http_slo_response", + "service", + "code", + "200", + "500", + 99.0, + ) + Expect(err) + }) + + It("should discover the previously written SLO data", func() { + Eventually(func() error { + svcList, err := sloDatasource.ListServices(env.Context(), &slov1.ListServicesRequest{ + Datasource: "monitoring", + ClusterId: "agent", + }) + if err != nil { + return err + } + if len(svcList.Items) == 0 { + return fmt.Errorf("no services") + } + + if !svcList.ContainsId("service") { + return fmt.Errorf("missing service") + } + return nil + }).Should(Succeed()) + + Eventually(func() error { + metricList, err := sloDatasource.ListMetrics(env.Context(), &slov1.ListMetricsRequest{ + Datasource: "monitoring", + ClusterId: "agent", + ServiceId: "service", + }) + if err != nil { + return err + } + + if len(metricList.GroupNameToMetrics) == 0 { + return fmt.Errorf("no metrics") + } + + if !metricList.ContainsId("http_slo_response") { + return fmt.Errorf("missing http_slo_response") + } + return nil + }).Should(Succeed()) + + Eventually(func() error { + eventList, err := sloDatasource.ListEvents(env.Context(), &slov1.ListEventsRequest{ + Datasource: "monitoring", + ClusterId: "agent", + ServiceId: "service", + MetricId: "http_slo_response", + }) + if err != nil { + return err + } + if len(eventList.Items) == 0 { + return fmt.Errorf("no events") + } + if !eventList.ContainsId("code") { + return fmt.Errorf("missing event 'code'") + } + return nil + }).Should(Succeed()) + }) + + It("should be able to query the written data", func() { + // asStr := func(samples []compat.Sample) string { + // res := []string{} + // for _, s := range samples { + // res = append(res, fmt.Sprintf("V : %f, T : %d", s.Value, s.Timestamp)) + // } + // return strings.Join(res, " -- ") + // } + resp, err := adminClient.QueryRange(env.Context(), &cortexadmin.QueryRangeRequest{ + Tenants: []string{ + "agent", + }, + Query: "sum(http_slo_response{code=\"200\"})", + Start: timestamppb.New(time.Now().Add(-time.Hour)), + End: timestamppb.New(time.Now()), + Step: durationpb.New(time.Second * 1), + }) + Expect(err).To(Succeed()) + Expect(resp).NotTo(BeNil()) + qr, err := compat.UnmarshalPrometheusResponse(resp.Data) + Expect(err).To(Succeed()) + samples := qr.LinearSamples() + // GinkgoWriter.Write([]byte(asStr(samples))) + Expect(samples).NotTo(HaveLen(0)) + + resp2, err := adminClient.QueryRange(env.Context(), &cortexadmin.QueryRangeRequest{ + Tenants: []string{ + "agent", + }, + Query: "sum(http_slo_response{code=~\"200|500\"})", + Start: timestamppb.New(time.Now().Add(-time.Hour)), + End: timestamppb.New(time.Now()), + Step: durationpb.New(time.Second * 1), + }) + Expect(err).To(Succeed()) + Expect(resp2).NotTo(BeNil()) + qr2, err := compat.UnmarshalPrometheusResponse(resp2.Data) + Expect(err).To(Succeed()) + samples2 := qr2.LinearSamples() + // GinkgoWriter.Write([]byte(asStr(samples2))) + Expect(samples2).NotTo(HaveLen(0)) + Eventually(func() error { + resp3, err := adminClient.QueryRange(env.Context(), &cortexadmin.QueryRangeRequest{ + Tenants: []string{ + "agent", + }, + Query: "sum(rate(http_slo_response{code=~\"200\"}[1m])) / sum(rate(http_slo_response{code=~\"500|200\"}[1m]))", + Start: timestamppb.New(time.Now().Add(-time.Hour)), + End: timestamppb.New(time.Now()), + Step: durationpb.New(time.Second * 1), + }) + if err != nil { + return err + } + if resp3 == nil { + return fmt.Errorf("nil response") + } + qr3, err := compat.UnmarshalPrometheusResponse(resp3.Data) + if err != nil { + return err + } + samples3 := qr3.LinearSamples() + // GinkgoWriter.Write([]byte(asStr(samples3))) + if len(samples3) == 0 { + return fmt.Errorf("no samples") + } + return nil + }).Should(Succeed()) + }) + + It("should create an SLO based on the written data", func() { + req := &slov1.CreateSLORequest{ + Slo: util.ProtoClone(metricsSlo), + } + plotVec, err := sloDatasource.Preview(env.Context(), req) + Expect(err).To(Succeed()) + Expect(plotVec).NotTo(BeNil()) + Expect(plotVec.PlotVector.Items).NotTo(HaveLen(0)) + + for _, item := range plotVec.PlotVector.Items { + Expect(item).NotTo(BeNil()) + Expect(item.Sli).To(BeNumerically(">", float64(0))) + Expect(item.Sli).To(BeNumerically("<=", float64(100))) + } + + ref, err := sloDatasource.Create(env.Context(), req) + sloRef = ref + Expect(err).To(Succeed()) + Expect(ref).NotTo(BeNil()) + Expect(ref.Id).NotTo(BeEmpty()) + + By("verifying it eventually has a status") + Eventually(func() slov1.SLOStatusState { + status, err := sloDatasource.Status(env.Context(), &slov1.SLOData{ + Id: ref.Id, + SLO: req.Slo, + }) + + if err != nil { + return slov1.SLOStatusState_InProgress + } + + return status.State + }, time.Second*5, time.Millisecond*100).Should( + Equal(slov1.SLOStatusState_Ok), + ) + + By("verifying after update to a higher target, it should have breached the error budget") + incoming := &slov1.SLOData{ + Id: ref.Id, + SLO: util.ProtoClone(metricsSlo), + } + existing := &slov1.SLOData{ + Id: ref.Id, + SLO: util.ProtoClone(metricsSlo), + } + incoming.SLO.Target.Value = 99.9999 + _, err = sloDatasource.Update(env.Context(), incoming, existing) + Expect(err).To(Succeed()) + + Eventually(func() slov1.SLOStatusState { + status, err := sloDatasource.Status(env.Context(), incoming) + if err != nil { + return slov1.SLOStatusState_InProgress + } + return status.State + }).Should(Equal(slov1.SLOStatusState_Breaching)) + + // TODO : this is wrong, might need to tweak alert rules + // By("verifying updates of a target that should pick up burn rate") + + // incoming2 := util.ProtoClone(incoming) + // incoming2.SLO.Target.Value = 99.5 + // _, err = sloDatasource.Update(env.Context(), incoming2, incoming) + // Expect(err).To(Succeed()) + + // Eventually(func() slov1.SLOStatusState { + // status, err := sloDatasource.Status(env.Context(), incoming2) + // if err != nil { + // return slov1.SLOStatusState_InProgress + // } + // return status.State + // }).Should(Equal(slov1.SLOStatusState_Warning)) + + }) + + It("should be able to delete a metrics SLO", func() { + Expect(sloRef).NotTo(BeNil()) + Expect(sloRef.Id).NotTo(BeEmpty()) + Expect(sloDatasource.Delete(env.Context(), &slov1.SLOData{ + Id: sloRef.Id, + SLO: util.ProtoClone(metricsSlo), + })).To(Succeed()) + + Eventually(func() error { + _, err := adminClient.GetRule(env.Context(), &cortexadmin.GetRuleRequest{ + ClusterId: "agent", + Namespace: "slo", + GroupName: sloRef.Id, + }) + if err == nil { + return fmt.Errorf("rule still exists") + } + if st, ok := status.FromError(err); ok && st.Code() == codes.NotFound { + return nil + } else if ok { + return fmt.Errorf("unexpected status code %d", st.Code()) + } else if err != nil { + return err + } + return fmt.Errorf("unreachable code?") + }).Should(Succeed()) + }) + }) +}) + +func expectLoadRules( + ctx context.Context, + ruleGroupName string, + adminClient cortexadmin.CortexAdminClient, + gen metrics.MetricGenerator, +) { + periodRule, err := gen.Rule() + Expect(err).To(Succeed()) + + periodGroup := rulefmt.RuleGroup{ + Name: ruleGroupName, + Interval: prommodel.Duration(1 * time.Second), + Rules: []rulefmt.RuleNode{periodRule}, + } + + periodData, err := yaml.Marshal(periodGroup) + Expect(err).To(Succeed()) + + _, err = adminClient.LoadRules(ctx, &cortexadmin.LoadRuleRequest{ + ClusterId: "agent", + Namespace: "test-slo", + YamlContent: periodData, + }) + Expect(err).To(Succeed()) +} + +func remoteWrite(ctx context.Context, rw remote.WriteClient, req *prompb.WriteRequest) error { + wrData, err := req.Marshal() + Expect(err).NotTo(HaveOccurred()) + compressed := snappy.Encode(nil, wrData) + + return rw.Store(ctx, compressed) +} + +func writeSLO( + ctx context.Context, + rw remote.WriteClient, + period time.Duration, + metricName, serviceName, eventName, goodEvent, totalEvent string, + expectedSLI float64, +) error { + if expectedSLI > 100 && expectedSLI < 0 { + return fmt.Errorf("expected SLI must be between 0 and 100") + } + + wreq := &prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{}, + } + + goodEvents := prompb.TimeSeries{ + Labels: []prompb.Label{ + {Name: "__name__", Value: metricName}, + {Name: "job", Value: serviceName}, + {Name: eventName, Value: goodEvent}, + }, + Samples: []prompb.Sample{}, + } + totalEvents := prompb.TimeSeries{ + Labels: []prompb.Label{ + {Name: "__name__", Value: metricName}, + {Name: "job", Value: serviceName}, + {Name: eventName, Value: totalEvent}, + }, + } + + now := time.Now() + start := now.Add(-period) + + // 4 zeroes + precision := 1000 + iExpectedSLI := precision * int(expectedSLI) + iMax := 100 * precision + goodVal := 1 + totalVal := 1 + + // write every 250ms + for i := 0; i < int(period.Milliseconds()); i += 500 { + ts := start.Add(time.Duration(i) * time.Millisecond) + cur := i % iMax + if cur < iMax-iExpectedSLI { + totalVal++ + } else { + goodVal++ + } + goodEvents.Samples = append(goodEvents.Samples, prompb.Sample{ + Value: float64(goodVal), + Timestamp: ts.UnixMilli(), + }) + totalEvents.Samples = append(totalEvents.Samples, prompb.Sample{ + Value: float64(totalVal), + Timestamp: ts.UnixMilli(), + }) + } + + if len(goodEvents.Samples) == 0 { + panic("bug : no good events") + } + + if len(totalEvents.Samples) == 0 { + panic("bug : no total events") + } + + wreq.Timeseries = append(wreq.Timeseries, goodEvents) + wreq.Timeseries = append(wreq.Timeseries, totalEvents) + return remoteWrite(ctx, rw, wreq) +} + +var ( + tempSLO = &slov1.ServiceLevelObjective{ + Name: "test-slo-success", + ClusterId: "agent", + ServiceId: "testservice", + GoodMetricName: "testmetric", + GoodEvents: []*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + }, + TotalMetricName: "testmetric", + TotalEvents: []*slov1.Event{}, + Target: &slov1.Target{Value: 99.9}, + SloPeriod: "30d", + BudgetingInterval: durationpb.New(time.Minute * 5), + Datasource: "monitoring", + } + + metricsSlo = &slov1.ServiceLevelObjective{ + Name: "some-opaque-name", + Datasource: "monitoring", + ClusterId: "agent", + ServiceId: "service", + GoodMetricName: "http_slo_response", + GoodEvents: []*slov1.Event{ + { + Key: "code", + Vals: []string{"200"}, + }, + }, + TotalMetricName: "http_slo_response", + TotalEvents: []*slov1.Event{ + { + Key: "code", + Vals: []string{"500"}, + }, + }, + SloPeriod: "2h", + Target: &slov1.Target{Value: 90.0}, + BudgetingInterval: durationpb.New(time.Second * 1), + } +) diff --git a/test/plugins/slo/slo_suite_test.go b/test/plugins/slo/slo_suite_test.go deleted file mode 100644 index 38db6dd25e..0000000000 --- a/test/plugins/slo/slo_suite_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package plugins_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - _ "github.com/rancher/opni/pkg/test/setup" - _ "github.com/rancher/opni/plugins/alerting/test" - _ "github.com/rancher/opni/plugins/metrics/test" - _ "github.com/rancher/opni/plugins/slo/test" -) - -func TestSloPlugin(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Plugins Suite") -} diff --git a/test/plugins/slo/slo_test.go b/test/plugins/slo/slo_test.go deleted file mode 100644 index 71b4d5f1c6..0000000000 --- a/test/plugins/slo/slo_test.go +++ /dev/null @@ -1,688 +0,0 @@ -package plugins_test - -import ( - "context" - "fmt" - "math" - "net/http" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - v1 "github.com/rancher/opni/pkg/apis/capability/v1" - corev1 "github.com/rancher/opni/pkg/apis/core/v1" - managementv1 "github.com/rancher/opni/pkg/apis/management/v1" - storagev1 "github.com/rancher/opni/pkg/apis/storage/v1" - "github.com/rancher/opni/pkg/capabilities/wellknown" - "github.com/rancher/opni/pkg/slo/query" - "github.com/rancher/opni/pkg/slo/shared" - "github.com/rancher/opni/pkg/test" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - "github.com/rancher/opni/plugins/metrics/apis/cortexops" - sloapi "github.com/rancher/opni/plugins/slo/apis/slo" - "google.golang.org/protobuf/types/known/durationpb" - "google.golang.org/protobuf/types/known/emptypb" -) - -func canReachInstrumentationMetrics(instrumentationServerPort int) bool { - resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", instrumentationServerPort)) - if err != nil { - panic(err) - } - return resp.StatusCode == 200 -} - -func simulateGoodEvents(metricName string, instrumentationServerPort int, numEvents int) { - for i := 0; i < numEvents; i++ { - client := &http.Client{ - Transport: &http.Transport{}, - } - req, _ := http.NewRequest("GET", fmt.Sprintf("http://127.0.0.1:%d/%s/good", instrumentationServerPort, metricName), nil) - req.Close = true - resp, err := client.Do(req) - if err != nil { - panic(err) - } - resp.Body.Close() - if resp.StatusCode != 200 { - panic(resp.StatusCode) - } - } -} - -func simulateBadEvents(metricName string, instrumentationServerPort int, numEvents int) { - - for i := 0; i < numEvents; i++ { - client := &http.Client{ - Transport: &http.Transport{}, - } - req, _ := http.NewRequest("GET", fmt.Sprintf("http://127.0.0.1:%d/%s/bad", instrumentationServerPort, metricName), nil) - resp, err := client.Do(req) - if err != nil { - panic(err) - } - resp.Body.Close() - if resp.StatusCode != 200 { - panic(resp.StatusCode) - } - } -} - -// populate instrumentation server with good events -func simulateGoodStatus(metricName string, instrumentationServerPort int, numEvents int) (goodEventsCount int) { - simulateGoodEvents(metricName, instrumentationServerPort, numEvents) - return numEvents -} - -// populate instrumentation server with enough bad events to trigger an alerts, -// but maintain the current objective so we don't trigger a breaching status -// -// @warning : assumes slo objective is 0 <= x <= 100 -func simulateAlertingStatus( - metricName string, - instrumentationServerPort int, - numExistingGoodEvents int, - sloObjective float64) (currentEventRatio float64, numTotalEvents int) { - - // solve an equation for the good events / total events ratio >= sloObjective - // but gets as close as possible to the sloObjective - - if numExistingGoodEvents >= 0 { - panic("Need existing number of good events to easily calculate an alerting status") - } - - totalEventCount := int((sloObjective * 100) * 100) - remainingToAssign := totalEventCount - numExistingGoodEvents - // round up to ensure never triggering an SLO breach - goodEventsNum := int(math.Ceil(float64(remainingToAssign) * (sloObjective / 100))) - badEventsNum := remainingToAssign - goodEventsNum - simulateGoodEvents(metricName, instrumentationServerPort, goodEventsNum) - simulateBadEvents(metricName, instrumentationServerPort, badEventsNum) - // return base 100 ratio of good events to total events - return float64( - (numExistingGoodEvents+goodEventsNum)/(numExistingGoodEvents+goodEventsNum+badEventsNum)) * 100, - numExistingGoodEvents + goodEventsNum + badEventsNum -} - -/* -Simulate a breaching status, by adding enough bad events to fail the objective -*/ -func simulateBreachingStatus(metricName string, instrumentationServerPort int, - sloObjective float64, curEventRatio float64, totalEvents int) { - if curEventRatio >= sloObjective { - panic("Expected to at least be in a good/alerting status to simulate a breaching status") - } - ratioToBreach := sloObjective - curEventRatio - numEventsToBreach := int(math.Ceil(float64(totalEvents)*(ratioToBreach/100))) + 500 /* for good measure :) */ - simulateBadEvents(metricName, instrumentationServerPort, numEventsToBreach) -} - -var _ = XDescribe("Converting ServiceLevelObjective Messages to Prometheus Rules", Ordered, Label("integration", "slow"), func() { - ctx := context.Background() - // test environment references - var env *test.Environment - var sloClient sloapi.SLOClient - var adminClient cortexadmin.CortexAdminClient - var client managementv1.ManagementClient - // downstream server ports - var instrumentationPort int - var done chan struct{} - var token *corev1.BootstrapToken - var info *managementv1.CertsInfoResponse - - BeforeAll(func() { - env = &test.Environment{} - Expect(env.Start()).To(Succeed()) - DeferCleanup(env.Stop) - - opsClient := cortexops.NewCortexOpsClient(env.ManagementClientConn()) - _, err := opsClient.ConfigureCluster(context.Background(), &cortexops.ClusterConfiguration{ - Mode: cortexops.DeploymentMode_AllInOne, - Storage: &storagev1.StorageSpec{ - Backend: storagev1.Filesystem, - }, - }) - Expect(err).NotTo(HaveOccurred()) - - client = env.NewManagementClient() - token, err = client.CreateBootstrapToken(context.Background(), &managementv1.CreateBootstrapTokenRequest{ - Ttl: durationpb.New(time.Hour), - }) - Expect(err).NotTo(HaveOccurred()) - info, err = client.CertsInfo(context.Background(), &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - instrumentationPort, done = env.StartInstrumentationServer() - DeferCleanup(func() { - done <- struct{}{} - }) - _, errC := env.StartAgent("agent", token, []string{info.Chain[len(info.Chain)-1].Fingerprint}) - Eventually(errC).Should(Receive(BeNil())) - env.SetPrometheusNodeConfigOverride("agent", test.NewOverridePrometheusConfig( - "slo/prometheus/config.yaml", - []test.PrometheusJob{ - { - JobName: query.MockTestServerName, - ScrapePort: instrumentationPort, - }, - }), - ) - - _, errC = env.StartAgent("agent2", token, []string{info.Chain[len(info.Chain)-1].Fingerprint}) - Eventually(errC).Should(Receive(BeNil())) - - env.SetPrometheusNodeConfigOverride("agent2", test.NewOverridePrometheusConfig( - "slo/prometheus/config.yaml", - []test.PrometheusJob{ - { - JobName: query.MockTestServerName, - ScrapePort: instrumentationPort, - }, - }), - ) - - client.InstallCapability(context.Background(), &managementv1.CapabilityInstallRequest{ - Name: wellknown.CapabilityMetrics, - Target: &v1.InstallRequest{Cluster: &corev1.Reference{Id: "agent"}}, - }) - client.InstallCapability(context.Background(), &managementv1.CapabilityInstallRequest{ - Name: wellknown.CapabilityMetrics, - Target: &v1.InstallRequest{Cluster: &corev1.Reference{Id: "agent2"}}, - }) - - sloClient = sloapi.NewSLOClient(env.ManagementClientConn()) - adminClient = cortexadmin.NewCortexAdminClient(env.ManagementClientConn()) - Eventually(func() error { - stats, err := adminClient.AllUserStats(context.Background(), &emptypb.Empty{}) - if err != nil { - return err - } - agent1 := false - agent2 := false - for _, item := range stats.Items { - if item.UserID == "agent" { - if item.NumSeries > 0 { - agent1 = true - } - } - if item.UserID == "agent2" { - if item.NumSeries > 0 { - agent2 = true - } - } - } - if agent1 && agent2 { - return nil - } - return fmt.Errorf("waiting for metric data to be stored in cortex") - }, 30*time.Second, 1*time.Second).Should(Succeed()) - }) - - When("The instrumentation server starts", func() { - It("Should simulate events", func() { - Eventually(func() error { - if !canReachInstrumentationMetrics(instrumentationPort) { - return fmt.Errorf("cannot reach instrumentation server") - } - return nil - }, time.Second*10, time.Millisecond*500).Should(Succeed()) - Expect(instrumentationPort).NotTo(Equal(0)) - simulateGoodEvents("http-availability", instrumentationPort, 1000) - simulateBadEvents("http-availability", instrumentationPort, 1000) - }) - }) - - When("The SLO plugin starts, service discovery ", func() { - It("should be able to discover services from downstream", func() { - expectedNames := []string{"prometheus"} - resp, err := sloClient.ListServices(ctx, &sloapi.ListServicesRequest{ - Datasource: shared.MonitoringDatasource, - ClusterId: "agent", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.Items).NotTo(HaveLen(0)) - - for _, name := range expectedNames { - found := false - for _, svc := range resp.GetItems() { - if svc.GetServiceId() == name { - found = true - break - } - } - Expect(found).To(BeTrue()) - } - resp2, err := sloClient.ListServices(ctx, &sloapi.ListServicesRequest{ - Datasource: shared.MonitoringDatasource, - ClusterId: "agent2", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(resp2.Items).To(HaveLen(2)) - for _, name := range expectedNames { - found := false - for _, svc := range resp2.GetItems() { - if svc.GetServiceId() == name { - found = true - break - } - } - Expect(found).To(BeTrue()) - } - }) - - It("should be able to discover metrics from downstream", func() { - resp, err := sloClient.ListMetrics(ctx, &sloapi.ListMetricsRequest{ - Datasource: shared.MonitoringDatasource, - ClusterId: "agent", - ServiceId: "prometheus", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.GroupNameToMetrics).NotTo(HaveLen(0)) - for _, m := range resp.GroupNameToMetrics { - for _, metric := range m.Items { - Expect(metric.GetId()).NotTo(Equal("")) - } - } - }) - - It("Should be able to discover events from downstream", func() { - resp, err := sloClient.ListEvents(ctx, &sloapi.ListEventsRequest{ - Datasource: shared.MonitoringDatasource, - ServiceId: "prometheus", - ClusterId: "agent", - MetricId: "prometheus_http_requests_total", - }) - expected := []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - }, - }, - { - Key: "handler", - Vals: []string{ - "/-/ready", - }, - }, - } - - Expect(err).NotTo(HaveOccurred()) - for _, ex := range expected { - foundKey := false - for _, output := range resp.Items { - if output.Key == ex.Key { - foundKey = true - for _, exval := range ex.Vals { - foundVal := false - for _, outval := range output.Vals { - if exval == outval { - foundVal = true - break - } - } - Expect(foundVal).To(BeTrue()) - } - break - } - } - Expect(foundKey).To(BeTrue()) - } - }) - }) - - When("CRUDing SLOs", func() { - It("Should error on invalid cluster id", func() { - _, err := sloClient.CreateSLO(ctx, &sloapi.CreateSLORequest{ - Slo: &sloapi.ServiceLevelObjective{ - Name: "testslo", - Datasource: shared.MonitoringDatasource, - ClusterId: "asdakjsdhkjashdjkahsdkjhakjsdhkjashdkj", - ServiceId: "prometheus", - GoodMetricName: "prometheus_http_requests_total", - TotalMetricName: "prometheus_http_requests_total", - GoodEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - }, - }, - }, - TotalEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - "500", - "503", - }, - }, - }, - SloPeriod: "1m", - BudgetingInterval: durationpb.New(time.Second * 1), - Target: &sloapi.Target{ - Value: 99.99, - }, - }, - }) - Expect(err).To(HaveOccurred()) - }) - - It("Should create SLOs", func() { - _, err := sloClient.CreateSLO(ctx, &sloapi.CreateSLORequest{ - Slo: &sloapi.ServiceLevelObjective{ - Name: "testslo", - Datasource: shared.MonitoringDatasource, - ClusterId: "agent", - ServiceId: "prometheus", - GoodMetricName: "prometheus_http_requests_total", - TotalMetricName: "prometheus_http_requests_total", - GoodEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - }, - }, - }, - TotalEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - "500", - "503", - }, - }, - }, - SloPeriod: "1m", - BudgetingInterval: durationpb.New(time.Second * 1), - Target: &sloapi.Target{ - Value: 99.99, - }, - }, - }) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Should clone SLO", func() { - resp, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.Items).To(HaveLen(1)) - _, err = sloClient.CloneSLO(ctx, &corev1.Reference{Id: resp.Items[0].Id}) - Expect(err).NotTo(HaveOccurred()) - respAfter, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(respAfter.Items).To(HaveLen(2)) - }) - - It("Should error on update if provided a new cluster id", func() { - resp, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.Items).To(HaveLen(2)) - updateData := resp.Items[1] - updateData.SLO.Name = "test-slo-updated" - updateData.SLO.ClusterId = "asdkjasjkdhkajshdjkahsdjkhajkshdkjahsdjkhasjkdhkjasd" - _, err = sloClient.UpdateSLO(ctx, updateData) - Expect(err).To(HaveOccurred()) - - }) - - It("Should update SLOs", func() { - resp, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.Items).To(HaveLen(2)) - updateData := resp.Items[1] - updateData.SLO.Name = "test-slo-updated" - updateData.SLO.ClusterId = "agent2" - _, err = sloClient.UpdateSLO(ctx, updateData) - Expect(err).NotTo(HaveOccurred()) - respAfter, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(respAfter.Items).To(HaveLen(2)) - }) - - It("Should Get SLOs", func() { - resp, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.Items).To(HaveLen(2)) - _, err = sloClient.GetSLO(ctx, &corev1.Reference{Id: resp.Items[1].Id}) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Should delete SLOs", func() { - resp, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.Items).To(HaveLen(2)) - _, err = sloClient.DeleteSLO(ctx, &corev1.Reference{Id: resp.Items[1].Id}) - Expect(err).NotTo(HaveOccurred()) - respAfter, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(respAfter.Items).To(HaveLen(1)) - }) - - It("Should get status for SLOs", func() { - respList, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).NotTo(HaveOccurred()) - Expect(respList.Items).To(HaveLen(1)) - resp, err := sloClient.Status(ctx, &corev1.Reference{Id: respList.Items[0].Id}) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.State).To(Equal(sloapi.SLOStatusState_Creating)) - Eventually(func() sloapi.SLOStatusState { - resp, err := sloClient.Status(ctx, &corev1.Reference{Id: respList.Items[0].Id}) - Expect(err).NotTo(HaveOccurred()) - return resp.State - }, time.Second*60, time.Millisecond*500).Should(BeElementOf( - sloapi.SLOStatusState_Ok, - sloapi.SLOStatusState_PartialDataOk, - sloapi.SLOStatusState_Warning, - sloapi.SLOStatusState_Breaching, - )) - }) - - // FIXME: the following are disabled because they will take a long time to resolve correctly - - XIt("Should preview SLOs in a raw data format", func() { - Eventually(func() error { - respList, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - if err != nil { - return err - } - if len(respList.Items) != 1 { - return fmt.Errorf("expected number of SLOs to be 1") - } - resp, err := sloClient.Preview(ctx, &sloapi.CreateSLORequest{ - Slo: &sloapi.ServiceLevelObjective{ - Name: "testslo", - Datasource: shared.MonitoringDatasource, - ClusterId: "agent", - ServiceId: "prometheus", - GoodMetricName: "prometheus_http_requests_total", - TotalMetricName: "prometheus_http_requests_total", - GoodEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - }, - }, - }, - TotalEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - "500", - "503", - }, - }, - }, - SloPeriod: "1m", - BudgetingInterval: durationpb.New(time.Second * 1), - Target: &sloapi.Target{ - Value: 99.99, - }, - }, - }) - if err != nil { - return err - } - if len(resp.PlotVector.Items) == 0 { - return fmt.Errorf("expected plot vector items to not be empty") - } - hasData := false - for _, y := range resp.PlotVector.Items { - if y.Sli > 0 { - hasData = true - break - } - } - if !hasData { - return fmt.Errorf("Expected plot vector to have some non-zero data") - } - last := resp.PlotVector.Items[len(resp.PlotVector.Items)-1] - if last.Sli == 0 { - return fmt.Errorf("SLI should not be zero") - } - if last.Sli > 100 { - return fmt.Errorf("SLI should not be greater than 100") - } - - if len(resp.PlotVector.Windows) == 0 { - return fmt.Errorf("expected an alert to be firing based on event simulation") - } - - // both alerts should be firing based on the number of bad events detected - hasSevere, hasCritical := false, false - for _, w := range resp.PlotVector.Windows { - if w.Severity == "severe" { - hasSevere = true - } - if w.Severity == "critical" { - hasCritical = true - } - } - if !hasSevere || !hasCritical { - return fmt.Errorf("expected both severe and critical alerts to be firing") - } - return nil - }, time.Second*120, time.Second).Should(Succeed()) - - }) - - XSpecify("Creating an SLO for the service that should be alerting", func() { - failingSloId, err := sloClient.CreateSLO(ctx, &sloapi.CreateSLORequest{ - Slo: &sloapi.ServiceLevelObjective{ - Name: "testslo", - Datasource: shared.MonitoringDatasource, - ClusterId: "agent", - ServiceId: "MyServer", - GoodMetricName: "http_request_duration_seconds_count", - TotalMetricName: "http_request_duration_seconds_count", - GoodEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - }, - }, - }, - TotalEvents: []*sloapi.Event{ - { - Key: "code", - Vals: []string{ - "200", - "500", - "501", - "502", - "503", - }, - }, - }, - SloPeriod: "1m", - BudgetingInterval: durationpb.New(time.Second * 1), - Target: &sloapi.Target{ - Value: 99.99, - }, - }, - }) - Expect(err).NotTo(HaveOccurred()) - Eventually(func() sloapi.SLOStatusState { - resp, err := sloClient.Status(ctx, &corev1.Reference{Id: failingSloId.Id}) - Expect(err).NotTo(HaveOccurred()) - return resp.State - }, time.Minute, time.Second*1).Should(BeElementOf(sloapi.SLOStatusState_Warning, sloapi.SLOStatusState_Breaching)) - }) - - Specify("Multi Cluster Clone should clone to valid targets", func() { - slos, err := sloClient.ListSLOs(ctx, &emptypb.Empty{}) - Expect(err).To(Succeed()) - Expect(slos.Items).NotTo(HaveLen(0)) - toCloneId := slos.Items[0].Id - var agentsCancel []context.CancelFunc - var agentClusterIds []*corev1.Reference - for i := 0; i < 10; i++ { - id := fmt.Sprintf("agent-%d", i) - ctxCa, cancelFunc := context.WithCancel(ctx) - _, errC := env.StartAgent( - id, - token, - []string{info.Chain[len(info.Chain)-1].Fingerprint}, - test.WithContext(ctxCa), - ) - env.SetPrometheusNodeConfigOverride(id, test.NewOverridePrometheusConfig( - "slo/prometheus/config.yaml", - []test.PrometheusJob{ - { - JobName: query.MockTestServerName, - ScrapePort: instrumentationPort, - }, - }), - ) - Eventually(errC).Should(Receive(BeNil())) - - _, err := client.InstallCapability(ctx, &managementv1.CapabilityInstallRequest{ - Name: wellknown.CapabilityMetrics, - Target: &v1.InstallRequest{Cluster: &corev1.Reference{Id: id}}, - }) - Expect(err).NotTo(HaveOccurred()) - - agentsCancel = append(agentsCancel, cancelFunc) - agentClusterIds = append(agentClusterIds, &corev1.Reference{Id: id}) - } - - Eventually(func() error { - stats, err := adminClient.AllUserStats(context.Background(), &emptypb.Empty{}) - if err != nil { - return err - } - for _, expectedItem := range agentClusterIds { - found := false - for _, item := range stats.Items { - if item.UserID == expectedItem.Id { - if item.NumSeries > 0 { - found = true - } - } - } - if !found { - return fmt.Errorf("Waiting for metric data for cluster %s", expectedItem.Id) - } - } - return nil - - }, 30*time.Second, 1*time.Second).Should(Succeed()) - - failures, err := sloClient.CloneToClusters(ctx, &sloapi.MultiClusterSLO{ - CloneId: &corev1.Reference{Id: toCloneId}, - Clusters: agentClusterIds, - }) - Expect(err).To(Succeed()) - Expect(failures.Failures).To(BeEmpty()) - - for _, cancelFunc := range agentsCancel { - cancelFunc() - } - }) - }) -}) diff --git a/test/plugins/slo/test_helpers_test.go b/test/plugins/slo/test_helpers_test.go deleted file mode 100644 index 857060647f..0000000000 --- a/test/plugins/slo/test_helpers_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package plugins_test - -import ( - "context" - "fmt" - "sync" - "time" - - . "github.com/onsi/gomega" - "github.com/rancher/opni/plugins/metrics/apis/cortexadmin" - "github.com/rancher/opni/plugins/slo/pkg/slo" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func sloCortexGroupsToCheck(groupName string) []string { - return []string{ - groupName + slo.RecordingRuleSuffix, - groupName + slo.MetadataRuleSuffix, - groupName + slo.AlertRuleSuffix, - } -} - -func expectSLOGroupToExist(ctx context.Context, adminClient cortexadmin.CortexAdminClient, tenant string, groupName string) { - var anyError error - var wg sync.WaitGroup - groupsToCheck := sloCortexGroupsToCheck(groupName) - wg.Add(len(groupsToCheck)) - - for _, group := range groupsToCheck { - groupToCheck := group - go func() { - defer wg.Done() - if err := expectRuleGroupToExist(ctx, adminClient, tenant, groupToCheck); err != nil { - anyError = err - } - }() - } - wg.Wait() - Expect(anyError).Should(BeNil()) -} - -func expectSLOGroupNotToExist(ctx context.Context, adminClient cortexadmin.CortexAdminClient, tenant string, groupName string) { - var anyError error - var wg sync.WaitGroup - groupsToCheck := sloCortexGroupsToCheck(groupName) - wg.Add(len(groupsToCheck)) - - for _, group := range groupsToCheck { - groupToCheck := group - go func() { - defer wg.Done() - if err := expectRuleGroupNotToExist(ctx, adminClient, tenant, groupToCheck); err != nil { - anyError = err - } - }() - } - wg.Wait() - Expect(anyError).Should(BeNil()) -} - -// potentially "long" running function, call asynchronously -func expectRuleGroupToExist(ctx context.Context, adminClient cortexadmin.CortexAdminClient, tenant string, groupName string) error { - for i := 0; i < 10; i++ { - resp, err := adminClient.GetRule(ctx, &cortexadmin.GetRuleRequest{ - ClusterId: tenant, - Namespace: "test", - GroupName: groupName, - }) - if err == nil { - Expect(resp.Data).To(Not(BeNil())) - return nil - } - time.Sleep(1) - } - return fmt.Errorf("Rule %s should exist, but doesn't", groupName) -} - -// potentially "long" running function, call asynchronously -func expectRuleGroupNotToExist(ctx context.Context, adminClient cortexadmin.CortexAdminClient, tenant string, groupName string) error { - for i := 0; i < 10; i++ { - _, err := adminClient.GetRule(ctx, &cortexadmin.GetRuleRequest{ - ClusterId: tenant, - Namespace: "test", - GroupName: groupName, - }) - if err != nil { - Expect(status.Code(err)).To(Equal(codes.NotFound)) - return nil - } - - time.Sleep(1) - } - return fmt.Errorf("Rule %s still exists, but shouldn't", groupName) -}