Merge upstream changes (#120)

* Fix exponentialMovingAverage panic

The panic was happening when the window size was greater than
the length of the values. Now if that happens no panics occur and
the average is returned. This matches graphiteweb.

* Add support for Graphite web toLowerCase function

* Add tests for toLowerCase function

* Update glue.go

* Update COMPATIBILITY.md to show support of toLowerCase

* Add support for Graphite web toUpperCase function

* Add tests for toUpperCase function

* Update glue.go

* Update COMPATIBILITY.md to include toUpperCase function

* Add support for Graphite web sumSeriesLists function

* Add tests for sumSeriesLists

* Update glue.go

* Update COMPATIBILITY.md to include sumSeriesLists

* Fix problems that I accidentally introduced while trying to resolve merge conflict

* Another attempt to fix previous problem

 Apparantly there is a directory that gen.go wasn't expecting and
 `go generate` in expr/functions produced `glue.go` file that didn't compile

 fix both generator and glue

* Adds Drone and Github workflow for Grafana's fork

This commit adds the files that are specific to Grafana's fork
that are not intended to be included in the upstream. This
files are already present in the commit history (they were deleted),
but this is an attempt of a fresh start on top of the upstream's `main`.
At the moment, this should the only commit we expect to skip when
sending our changes.

Co-authored-by: Nicolás Pazos <[email protected]>
Co-authored-by: Vladimir Smirnov <[email protected]>

Author: 3 people

COMPATIBILITY.md

@@ -149,7 +149,7 @@ limit: default value mismatch: got "INF", should be "Infinity" |
 | removeAbovePercentile | n: type mismatch: got integer, should be float |
 | removeAboveValue | n: type mismatch: got integer, should be float |
 | removeBelowPercentile | n: type mismatch: got integer, should be float |
-| removeBelowValue | n: type mismatch: got integer, should be float |
+| removeBelowValue | n: type mismatch: got integer, sho/usr/bin/nvidia-dockeruld be float |
 | round | precision: default value mismatch: got (empty), should be 0 |
 | scaleToSeconds | seconds: type mismatch: got integer, should be float |
 | smartSummarize | func: different amount of parameters, `[current rangeOf]` are missing
@@ -162,6 +162,7 @@ reverse: default value mismatch: got (empty), should be false |
 | timeShift | parameter not supported: alignDst |
 | timeSlice | endSliceAt: type mismatch: got interval, should be date |
 
+## Supported functions
 ## Supported functions
 | Function      | Carbonapi-only                                            |
 | :-------------|:--------------------------------------------------------- |
@@ -293,6 +294,7 @@ reverse: default value mismatch: got (empty), should be false |
 | substr(seriesList, start=0, stop=0) | no |
 | sum(*seriesLists) | no |
 | sumSeries(*seriesLists) | no |
+| sumSeriesLists(seriesListFirstPos, seriesListSecondPos) | no |
 | sumSeriesWithWildcards(seriesList, *position) | no |
 | summarize(seriesList, intervalString, func='sum', alignToFrom=False) | no |
 | threshold(value, label=None, color=None) | no |
@@ -301,6 +303,8 @@ reverse: default value mismatch: got (empty), should be false |
 | timeShift(seriesList, timeShift, resetEnd=True, alignDST=False) | no |
 | timeSlice(seriesList, startSliceAt, endSliceAt='now') | no |
 | timeStack(seriesList, timeShiftUnit='1d', timeShiftStart=0, timeShiftEnd=7) | no |
+| toLowerCase(seriesList)       | no |
+| toUpperCase(seriesList, *pos) | no |
 | transformNull(seriesList, default=0, referenceSeries=None) | no |
 | unique(*seriesLists) | no |
 | useSeriesAbove(seriesList, value, search, replace) | no |

expr/functions/exponentialMovingAverage/function.go

@@ -65,6 +65,10 @@ func (f *exponentialMovingAverage) Do(ctx context.Context, e parser.Expr, from,
 	var results []*types.MetricData
 	windowSize := n
 
+	if windowSize < 1 {
+		return nil, fmt.Errorf("invalid window size %d", windowSize)
+	}
+
 	start := from
 
 	arg, err := helper.GetSeriesArg(ctx, e.Args()[0], start, until, values)
@@ -83,20 +87,21 @@ func (f *exponentialMovingAverage) Do(ctx context.Context, e parser.Expr, from,
 
 		var vals []float64
 
-		if windowSize < 1 && windowSize > len(a.Values) {
-			return nil, fmt.Errorf("invalid window size %d", windowSize)
-		}
-
-		ema := consolidations.AggMean(a.Values[:windowSize])
+		if windowSize > len(a.Values) {
+			mean := consolidations.AggMean(a.Values)
+			vals = append(vals, helper.SafeRound(mean, 6))
+		} else {
+			ema := consolidations.AggMean(a.Values[:windowSize])
 
-		vals = append(vals, helper.SafeRound(ema, 6))
-		for _, v := range a.Values[windowSize:] {
-			if math.IsNaN(v) {
-				vals = append(vals, math.NaN())
-				continue
-			}
-			ema = constant*v + (1-constant)*ema
 			vals = append(vals, helper.SafeRound(ema, 6))
+			for _, v := range a.Values[windowSize:] {
+				if math.IsNaN(v) {
+					vals = append(vals, math.NaN())
+					continue
+				}
+				ema = constant*v + (1-constant)*ema
+				vals = append(vals, helper.SafeRound(ema, 6))
+			}
 		}
 
 		r.Tags[e.Target()] = fmt.Sprintf("%d", windowSize)

expr/functions/exponentialMovingAverage/function_test.go

@@ -33,6 +33,17 @@ func TestExponentialMovingAverage(t *testing.T) {
 				types.MakeMetricData("exponentialMovingAverage(metric1,3)", []float64{4, 6, 9, 11.5, 13.75, 15.875, 17.9375}, 1, 0),
 			},
 		},
+		{
+			// if the window is larger than the length of the values, the result should just be the average.
+			// this matches graphiteweb's behavior
+			"exponentialMovingAverage(metric1,100)",
+			map[parser.MetricRequest][]*types.MetricData{
+				{"metric1", 0, 1}: {types.MakeMetricData("metric1", []float64{1, 2, 3}, 1, startTime)},
+			},
+			[]*types.MetricData{
+				types.MakeMetricData("exponentialMovingAverage(metric1,100)", []float64{2}, 1, 0),
+			},
+		},
 	}
 
 	for _, tt := range tests {

expr/functions/glue.go

@@ -110,6 +110,8 @@ import (
 	"github.com/go-graphite/carbonapi/expr/functions/timeShiftByMetric"
 	"github.com/go-graphite/carbonapi/expr/functions/timeSlice"
 	"github.com/go-graphite/carbonapi/expr/functions/timeStack"
+	"github.com/go-graphite/carbonapi/expr/functions/toLowerCase"
+	"github.com/go-graphite/carbonapi/expr/functions/toUpperCase"
 	"github.com/go-graphite/carbonapi/expr/functions/transformNull"
 	"github.com/go-graphite/carbonapi/expr/functions/tukey"
 	"github.com/go-graphite/carbonapi/expr/functions/unique"
@@ -234,6 +236,8 @@ func New(configs map[string]string) {
 		{name: "timeShiftByMetric", filename: "timeShiftByMetric", order: timeShiftByMetric.GetOrder(), f: timeShiftByMetric.New},
 		{name: "timeSlice", filename: "timeSlice", order: timeSlice.GetOrder(), f: timeSlice.New},
 		{name: "timeStack", filename: "timeStack", order: timeStack.GetOrder(), f: timeStack.New},
+		{name: "toLowerCase", filename: "toLowerCase", order: toLowerCase.GetOrder(), f: toLowerCase.New},
+		{name: "toUpperCase", filename: "toUpperCase", order: toUpperCase.GetOrder(), f: toUpperCase.New},
 		{name: "transformNull", filename: "transformNull", order: transformNull.GetOrder(), f: transformNull.New},
 		{name: "tukey", filename: "tukey", order: tukey.GetOrder(), f: tukey.New},
 		{name: "unique", filename: "unique", order: unique.GetOrder(), f: unique.New},

expr/functions/seriesList/function.go

@@ -23,14 +23,18 @@ func GetOrder() interfaces.Order {
 func New(configFile string) []interfaces.FunctionMetadata {
 	res := make([]interfaces.FunctionMetadata, 0)
 	f := &seriesList{}
-	functions := []string{"divideSeriesLists", "diffSeriesLists", "multiplySeriesLists", "powSeriesLists"}
+	functions := []string{"divideSeriesLists", "diffSeriesLists", "multiplySeriesLists", "powSeriesLists", "sumSeriesLists"}
 	for _, n := range functions {
 		res = append(res, interfaces.FunctionMetadata{Name: n, F: f})
 	}
 	return res
 }
 
 func (f *seriesList) Do(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
+	if e.ArgsLen() < 2 {
+		return nil, parser.ErrMissingArgument
+	}
+
 	useConstant := false
 	useDenom := false
 
@@ -85,6 +89,8 @@ func (f *seriesList) Do(ctx context.Context, e parser.Expr, from, until int64, v
 		compute = func(l, r float64) float64 { return l - r }
 	case "powSeriesLists":
 		compute = math.Pow
+	case "sumSeriesLists":
+		compute = func(l, r float64) float64 { return l + r }
 	}
 
 	if useConstant {
@@ -306,5 +312,33 @@ func (f *seriesList) Description() map[string]types.FunctionDescription {
 			TagsChange:   true, // name tag changed
 			ValuesChange: true, // values changed
 		},
+		"sumSeriesLists": {
+			Description: "Iterates over a two lists and subtracts series lists 2 through n from series 1 list1[0] to list2[0], list1[1] to list2[1] and so on. \n The lists will need to be the same length\nCarbonAPI-specific extension allows to specify default value as 3rd optional argument in case series doesn't exist or value is missing Example:\n\n.. code-block:: none\n\n  &target=sumSeriesLists(mining.{carbon,graphite,diamond}.extracted,mining.{carbon,graphite,diamond}.shipped)\n\n",
+			Function:    "sumSeriesLists(seriesListFirstPos, seriesListSecondPos)",
+			Group:       "Combine",
+			Module:      "graphite.render.functions.custom",
+			Name:        "sumSeriesLists",
+			Params: []types.FunctionParam{
+				{
+					Name:     "seriesListFirstPos",
+					Required: true,
+					Type:     types.SeriesList,
+				},
+				{
+					Name:     "seriesListSecondPos",
+					Required: true,
+					Type:     types.SeriesList,
+				},
+				{
+					Name:     "default",
+					Required: false,
+					Type:     types.Float,
+				},
+			},
+			SeriesChange: true, // function aggregate metrics or change series items count
+			NameChange:   true, // name changed
+			TagsChange:   true, // name tag changed
+			ValuesChange: true, // values changed
+		},
 	}
 }

expr/functions/seriesList/function_test.go

@@ -35,6 +35,15 @@ func TestFunction(t *testing.T) {
 			[]*types.MetricData{types.MakeMetricData("diffSeries(metric1,metric2)",
 				[]float64{-1, math.NaN(), math.NaN(), math.NaN(), 4, 6}, 1, now32)},
 		},
+		{
+			"sumSeriesLists(metric1,metric2)",
+			map[parser.MetricRequest][]*types.MetricData{
+				{"metric1", 0, 1}: {types.MakeMetricData("metric1", []float64{1, math.NaN(), math.NaN(), 3, 4, 12}, 1, now32)},
+				{"metric2", 0, 1}: {types.MakeMetricData("metric2", []float64{2, math.NaN(), 3, math.NaN(), 0, 6}, 1, now32)},
+			},
+			[]*types.MetricData{types.MakeMetricData("sumSeries(metric1,metric2)",
+				[]float64{3, math.NaN(), math.NaN(), math.NaN(), 4, 18}, 1, now32)},
+		},
 	}
 
 	for _, tt := range tests {
@@ -109,6 +118,20 @@ func TestSeriesListMultiReturn(t *testing.T) {
 				"diffSeries(metric1,metric1)": {types.MakeMetricData("diffSeries(metric1,metric1)", []float64{0, 0, 0, 0, 0}, 1, now32)},
 			},
 		},
+		{
+			"sumSeriesLists(metric[12],metric[12])",
+			map[parser.MetricRequest][]*types.MetricData{
+				{"metric[12]", 0, 1}: {
+					types.MakeMetricData("metric1", []float64{1, 2, 3, 4, 5}, 1, now32),
+					types.MakeMetricData("metric2", []float64{2, 4, 6, 8, 10}, 1, now32),
+				},
+			},
+			"sumSeriesListSameGroups",
+			map[string][]*types.MetricData{
+				"sumSeries(metric1,metric1)": {types.MakeMetricData("sumSeries(metric1,metric1)", []float64{2, 4, 6, 8, 10}, 1, now32)},
+				"sumSeries(metric2,metric2)": {types.MakeMetricData("sumSeries(metric2,metric2)", []float64{4, 8, 12, 16, 20}, 1, now32)},
+			},
+		},
 	}
 
 	for _, tt := range tests {

expr/functions/toLowerCase/function.go

@@ -0,0 +1,96 @@
+package toLowerCase
+
+import (
+	"context"
+	"github.com/go-graphite/carbonapi/expr/helper"
+	"strings"
+
+	"github.com/go-graphite/carbonapi/expr/interfaces"
+	"github.com/go-graphite/carbonapi/expr/types"
+	"github.com/go-graphite/carbonapi/pkg/parser"
+)
+
+type toLowerCase struct {
+	interfaces.FunctionBase
+}
+
+func GetOrder() interfaces.Order {
+	return interfaces.Any
+}
+
+func New(configFile string) []interfaces.FunctionMetadata {
+	res := make([]interfaces.FunctionMetadata, 0)
+	f := &toLowerCase{}
+	functions := []string{"toLowerCase"}
+	for _, n := range functions {
+		res = append(res, interfaces.FunctionMetadata{Name: n, F: f})
+	}
+	return res
+}
+
+// toLowerCase(seriesList, *pos)
+func (f *toLowerCase) Do(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
+	args, err := helper.GetSeriesArg(ctx, e.Args()[0], from, until, values)
+	if err != nil {
+		return nil, err
+	}
+
+	var pos []int
+
+	if e.ArgsLen() >= 2 {
+		pos, err = e.GetIntArgs(1)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	results := make([]*types.MetricData, 0, len(args)+1)
+
+	for _, a := range args {
+		r := a.CopyLink()
+
+		if len(pos) == 0 {
+			r.Name = strings.ToLower(a.Name)
+		} else {
+			for _, i := range pos {
+				if i < 0 { // Handle negative indices by indexing backwards
+					i = len(r.Name) + i
+				}
+				lowered := strings.ToLower(string(r.Name[i]))
+				r.Name = r.Name[:i] + lowered + r.Name[i+1:]
+			}
+		}
+
+		results = append(results, r)
+	}
+
+	return results, nil
+}
+
+// Description is auto-generated description, based on output of https://github.com/graphite-project/graphite-web
+func (f *toLowerCase) Description() map[string]types.FunctionDescription {
+	return map[string]types.FunctionDescription{
+		"toLowerCase": {
+			Description: "Takes one metric or a wildcard seriesList and lowers the case of each letter. \n Optionally, a letter position to lower case can be specified, in which case only the letter at the specified position gets lower-cased.\n The position parameter may be given multiple times. The position parameter may be negative to define a position relative to the end of the metric name.",
+			Function:    "toLowerCase(seriesList, *pos)",
+			Group:       "Alias",
+			Module:      "graphite.render.functions",
+			Name:        "toLowerCase",
+			Params: []types.FunctionParam{
+				{
+					Name:     "seriesList",
+					Required: true,
+					Type:     types.SeriesList,
+				},
+				{
+					Multiple: true,
+					Name:     "pos",
+					Type:     types.Node,
+					Required: false,
+				},
+			},
+			NameChange:   true, // name changed
+			ValuesChange: true, // values changed
+		},
+	}
+}

expr/functions/toLowerCase/function_test.go

@@ -0,0 +1,69 @@
+package toLowerCase
+
+import (
+	"math"
+	"testing"
+	"time"
+
+	"github.com/go-graphite/carbonapi/expr/helper"
+	"github.com/go-graphite/carbonapi/expr/metadata"
+	"github.com/go-graphite/carbonapi/expr/types"
+	"github.com/go-graphite/carbonapi/pkg/parser"
+	th "github.com/go-graphite/carbonapi/tests"
+)
+
+func init() {
+	md := New("")
+	evaluator := th.EvaluatorFromFunc(md[0].F)
+	metadata.SetEvaluator(evaluator)
+	helper.SetEvaluator(evaluator)
+	for _, m := range md {
+		metadata.RegisterFunction(m.Name, m.F)
+	}
+}
+
+func TestToLowerCaseFunction(t *testing.T) {
+	now32 := int64(time.Now().Unix())
+
+	tests := []th.EvalTestItem{
+		{
+			"toLowerCase(METRIC.TEST.FOO)",
+			map[parser.MetricRequest][]*types.MetricData{
+				{"METRIC.TEST.FOO", 0, 1}: {types.MakeMetricData("METRIC.TEST.FOO", []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)},
+			},
+			[]*types.MetricData{types.MakeMetricData("metric.test.foo",
+				[]float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)},
+		},
+		{
+			"toLowerCase(METRIC.TEST.FOO,7)",
+			map[parser.MetricRequest][]*types.MetricData{
+				{"METRIC.TEST.FOO", 0, 1}: {types.MakeMetricData("METRIC.TEST.FOO", []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)},
+			},
+			[]*types.MetricData{types.MakeMetricData("METRIC.tEST.FOO",
+				[]float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)},
+		},
+		{
+			"toLowerCase(METRIC.TEST.FOO,-3)",
+			map[parser.MetricRequest][]*types.MetricData{
+				{"METRIC.TEST.FOO", 0, 1}: {types.MakeMetricData("METRIC.TEST.FOO", []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)},
+			},
+			[]*types.MetricData{types.MakeMetricData("METRIC.TEST.fOO",
+				[]float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)},
+		},
+		{
+			"toLowerCase(METRIC.TEST.FOO,0,7,12)",
+			map[parser.MetricRequest][]*types.MetricData{
+				{"METRIC.TEST.FOO", 0, 1}: {types.MakeMetricData("METRIC.TEST.FOO", []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)},
+			},
+			[]*types.MetricData{types.MakeMetricData("mETRIC.tEST.fOO",
+				[]float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)},
+		},
+	}
+
+	for _, tt := range tests {
+		testName := tt.Target
+		t.Run(testName, func(t *testing.T) {
+			th.TestEvalExpr(t, &tt)
+		})
+	}
+}