diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 36d86d2..23d4ab1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/fsfe/reuse-tool - rev: v1.0.0 + rev: v4.0.3 hooks: - id: reuse stages: diff --git a/internal/k8s/watcher.go b/internal/k8s/watcher.go index dbe0734..819d710 100644 --- a/internal/k8s/watcher.go +++ b/internal/k8s/watcher.go @@ -5,6 +5,7 @@ package k8s import ( + "context" "fmt" "github.com/rs/zerolog/log" "github.com/telekom/quasar/internal/config" @@ -12,6 +13,7 @@ import ( "github.com/telekom/quasar/internal/mongo" "github.com/telekom/quasar/internal/store" "github.com/telekom/quasar/internal/utils" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/client-go/dynamic" "k8s.io/client-go/tools/cache" @@ -67,6 +69,8 @@ func NewResourceWatcher( DeleteFunc: watcher.delete, }) + go watcher.collectMetrics(client, resourceConfig) + return &watcher, err } @@ -147,3 +151,29 @@ func (w *ResourceWatcher) Start() { func (w *ResourceWatcher) Stop() { close(w.stopChan) } + +func (w *ResourceWatcher) collectMetrics(client dynamic.Interface, resourceConfig *config.ResourceConfiguration) { + if err := recover(); err != nil { + log.Error().Msgf("Recovered from %s during kubernetes metric collection", err) + return + } + + for { + list, err := client.Resource(resourceConfig.GetGroupVersionResource()). + Namespace(resourceConfig.Kubernetes.Namespace). + List(context.Background(), v1.ListOptions{}) + + if err != nil { + log.Error().Err(err).Fields(map[string]any{ + "resource": resourceConfig.GetCacheName(), + }).Msg("Could not resource count") + + time.Sleep(15 * time.Second) + continue + } + + var gaugeName = resourceConfig.GetCacheName() + "_kubernetes_count" + metrics.GetOrCreateCustom(gaugeName).WithLabelValues().Set(float64(len(list.Items))) + time.Sleep(15 * time.Second) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 3bd37ba..71b87db 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -47,3 +47,26 @@ func GetOrCreate(resourceConfig *config.ResourceConfiguration) *prometheus.Gauge return gauge } + +func GetOrCreateCustom(name string) *prometheus.GaugeVec { + var gaugeName = strings.ToLower(name) + + gauge, ok := gauges[name] + if !ok { + gauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: fmt.Sprintf(gaugeName), + }, []string{}) + + gauges[gaugeName] = gauge + if err := registry.Register(gauge); err != nil { + log.Error().Err(err). + Fields(map[string]any{ + "name": fmt.Sprintf("%s_%s", namespace, gaugeName), + }). + Msg("Could not create metric") + } + } + + return gauge +} diff --git a/internal/store/hazelcast.go b/internal/store/hazelcast.go index 7dd548d..a29d22a 100644 --- a/internal/store/hazelcast.go +++ b/internal/store/hazelcast.go @@ -10,9 +10,11 @@ import ( "github.com/hazelcast/hazelcast-go-client/serialization" "github.com/rs/zerolog/log" "github.com/telekom/quasar/internal/config" + "github.com/telekom/quasar/internal/metrics" "github.com/telekom/quasar/internal/mongo" "github.com/telekom/quasar/internal/utils" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "time" ) type HazelcastStore struct { @@ -63,6 +65,8 @@ func (s *HazelcastStore) InitializeResource(resourceConfig *config.ResourceConfi }).Err(err).Msg("Could not create hazelcast index") } } + + go s.collectMetrics(resourceConfig.GetCacheName()) } func (s *HazelcastStore) OnAdd(obj *unstructured.Unstructured) { @@ -133,3 +137,32 @@ func (s *HazelcastStore) getMap(obj *unstructured.Unstructured) *hazelcast.Map { return cacheMap } + +func (s *HazelcastStore) collectMetrics(resourceName string) { + if err := recover(); err != nil { + log.Error().Msgf("Recovered from %s during hazelcast metric collection", err) + return + } + + for { + hzMap, err := s.client.GetMap(context.Background(), resourceName) + if err != nil { + log.Error().Err(err).Fields(map[string]any{ + "map": hzMap.Name(), + }).Msg("Could not collect data") + } + + size, err := hzMap.Size(context.Background()) + if err != nil { + log.Error().Err(err).Fields(map[string]any{ + "map": hzMap.Name(), + }).Msg("Could not retrieve size") + + time.Sleep(15 * time.Second) + continue + } + + metrics.GetOrCreateCustom(resourceName + "_hazelcast_count").WithLabelValues().Set(float64(size)) + time.Sleep(15 * time.Second) + } +}