allow not published metrics (implement only submetric support)

Signed-off-by: Markus Blaschke <[email protected]>

Author: mblaschke

example.yaml

@@ -3,6 +3,10 @@ queries:
     # name of metric
   - metric: azure_resources
 
+    # skip metric publishing
+    # and only publish sub metrics rows (and use configuration only for submetrics)
+    # publish: false
+
     # Azure ResourceGraph query
     query: |-
       Resources

kusto/config.go

@@ -45,6 +45,7 @@ type (
 		Fields       []ConfigQueryMetricField `yaml:"fields"`
 		Labels       map[string]string        `yaml:"labels"`
 		DefaultField ConfigQueryMetricField   `yaml:"defaultField"`
+		Publish      *bool                    `yaml:"publish"`
 	}
 
 	ConfigQueryMetricField struct {
@@ -111,6 +112,14 @@ func (c *ConfigQueryMetric) Validate() error {
 	return nil
 }
 
+func (c *ConfigQueryMetric) IsPublished() bool {
+	if c.Publish != nil {
+		return *c.Publish
+	}
+
+	return true
+}
+
 func (c *ConfigQueryMetricField) Validate() error {
 	if c.Name == "" {
 		return errors.New("no field name set")

kusto/parse.go

@@ -64,6 +64,7 @@ func BuildPrometheusMetricList(name string, metricConfig ConfigQueryMetric, row
 					}
 				}
 
+				// check if metric should be skipped
 				if isNewMetricRow {
 					// save as own metric
 					if _, ok := list[fieldConfig.Metric]; !ok {
@@ -123,11 +124,13 @@ func BuildPrometheusMetricList(name string, metricConfig ConfigQueryMetric, row
 	}
 
 	// add main metrics
-	for metricName, metricRow := range mainMetrics {
-		if _, ok := list[metricName]; !ok {
-			list[metricName] = []MetricRow{}
+	if metricConfig.IsPublished() {
+		for metricName, metricRow := range mainMetrics {
+			if _, ok := list[metricName]; !ok {
+				list[metricName] = []MetricRow{}
+			}
+			list[metricName] = append(list[metricName], *metricRow)
 		}
-		list[metricName] = append(list[metricName], *metricRow)
 	}
 
 	// add id labels

kusto/parse_test.go

@@ -2,7 +2,6 @@ package kusto
 
 import (
 	"encoding/json"
-	"fmt"
 	"github.com/prometheus/client_golang/prometheus"
 	"gopkg.in/yaml.v2"
 	"testing"
@@ -66,10 +65,67 @@ defaultField:
 
 	metricList := BuildPrometheusMetricList(queryConfig.Metric, queryConfig.MetricConfig, resultRow)
 
-	fmt.Println(metricList)
-	if len(metricList) != 2 {
-		t.Fatalf(`metric count not valid, expected: %v, found: %v`, 2, len(metricList))
-	}
+	metricTestSuite := testingMetricResult{t: t, list: metricList}
+	metricTestSuite.assertMetricNames(2)
+
+	metricTestSuite.assertMetric("azure_testing")
+	metricTestSuite.metric("azure_testing").assertRowCount(1)
+	metricTestSuite.metric("azure_testing").row(0).assertLabels("id", "example")
+	metricTestSuite.metric("azure_testing").row(0).assertLabel("id", "foobar")
+	metricTestSuite.metric("azure_testing").row(0).assertLabel("example", "barfoo")
+	metricTestSuite.metric("azure_testing").row(0).assertValue(20)
+
+	metricTestSuite.assertMetric("azure_testing_value")
+	metricTestSuite.metric("azure_testing_value").assertRowCount(2)
+	metricTestSuite.metric("azure_testing_value").row(0).assertLabels("id", "scope")
+	metricTestSuite.metric("azure_testing_value").row(0).assertLabel("id", "foobar")
+	metricTestSuite.metric("azure_testing_value").row(0).assertLabel("scope", "one")
+	metricTestSuite.metric("azure_testing_value").row(0).assertValue(13)
+
+	metricTestSuite.metric("azure_testing_value").row(1).assertLabels("id", "scope")
+	metricTestSuite.metric("azure_testing_value").row(1).assertLabel("id", "foobar")
+	metricTestSuite.metric("azure_testing_value").row(1).assertLabel("scope", "two")
+	metricTestSuite.metric("azure_testing_value").row(1).assertValue(12)
+}
+
+func TestMetricRowParsingWithSubMetrics(t *testing.T) {
+	resultRow := parseResourceGraphJsonToResultRow(t, `{
+"name": "foobar",
+"count_": 20,
+"valueA": 13,
+"valueB": 12,
+"should-not-exists": "testing"
+}`)
+
+	queryConfig := parseMetricConfig(t, `
+metric: azure_testing
+labels:
+  example: barfoo
+fields:
+- name: name
+  target: id
+  type: id
+
+- name: count_
+  type: value
+
+- name: valueA
+  metric: azure_testing_value
+  type: value
+  labels:
+    scope: one
+
+- name: valueB
+  metric: azure_testing_value
+  type: value
+  labels:
+    scope: two
+
+defaultField:
+  type: ignore
+`)
+
+	metricList := BuildPrometheusMetricList(queryConfig.Metric, queryConfig.MetricConfig, resultRow)
 
 	metricTestSuite := testingMetricResult{t: t, list: metricList}
 	metricTestSuite.assertMetricNames(2)
@@ -99,7 +155,69 @@ defaultField:
 		secondRow.assertLabel("scope", "two")
 		secondRow.assertValue(12)
 	}
+}
+
+func TestMetricRowParsingWithSubMetricsWithDisabledMainMetric(t *testing.T) {
+	resultRow := parseResourceGraphJsonToResultRow(t, `{
+"name": "foobar",
+"count_": 20,
+"valueA": 13,
+"valueB": 12,
+"should-not-exists": "testing"
+}`)
+
+	queryConfig := parseMetricConfig(t, `
+metric: "azure_testing"
+publish: false
+labels:
+  example: barfoo
+fields:
+- name: name
+  target: id
+  type: id
+
+- name: count_
+  type: value
+
+- name: valueA
+  metric: azure_testing_value
+  type: value
+  labels:
+    scope: one
+
+- name: valueB
+  metric: azure_testing_value
+  type: value
+  labels:
+    scope: two
+
+defaultField:
+  type: ignore
+`)
+
+	metricList := BuildPrometheusMetricList(queryConfig.Metric, queryConfig.MetricConfig, resultRow)
+
+	metricTestSuite := testingMetricResult{t: t, list: metricList}
+	metricTestSuite.assertMetricNames(1)
+
+	metricTestSuite.assertMetric("azure_testing_value")
+	metricTestSuite.metric("azure_testing_value").assertRowCount(2)
+
+	firstRow := metricTestSuite.metric("azure_testing_value").findRowByLabels(prometheus.Labels{"scope": "one"})
+	{
+		firstRow.assertLabels("id", "scope")
+		firstRow.assertLabel("id", "foobar")
+		firstRow.assertLabel("scope", "one")
+		firstRow.assertValue(13)
+	}
 
+	secondRow := metricTestSuite.metric("azure_testing_value").findRowByLabels(prometheus.Labels{"scope": "two"})
+	{
+		secondRow.assertLabels("id", "scope")
+		secondRow.assertLabel("id", "foobar")
+		secondRow.assertLabel("scope", "two")
+		secondRow.assertValue(12)
+	}
 }
 
 func parseResourceGraphJsonToResultRow(t *testing.T, data string) map[string]interface{} {