diff --git a/.drone/Tests.Dockerfile b/.drone/Tests.Dockerfile new file mode 100644 index 000000000..8cf04af1f --- /dev/null +++ b/.drone/Tests.Dockerfile @@ -0,0 +1,2 @@ +FROM golang:alpine as builder +RUN apk --no-cache add make gcc git cairo-dev musl-dev \ No newline at end of file diff --git a/.drone/cmd/coverage/main.go b/.drone/cmd/coverage/main.go new file mode 100644 index 000000000..f8d425251 --- /dev/null +++ b/.drone/cmd/coverage/main.go @@ -0,0 +1,98 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + "sort" + "strconv" + "strings" +) + +type aggregation struct { + NumStmts int + NumCoveredStmts int +} + +func (a *aggregation) CoveragePct() string { + pct := (float64(a.NumCoveredStmts) / float64(a.NumStmts)) * 100 + return fmt.Sprintf("%.1f%%", pct) +} + +func main() { + covFilename := flag.String("f", "coverage.out", "Output of `go test -coverprofile=coverage.out ./...`") + flag.Parse() + + f, err := os.Open(*covFilename) + if err != nil { + panic(err) + } + + agg := make(map[string]*aggregation) + + s := bufio.NewScanner(f) + s.Split(bufio.ScanLines) + + s.Scan() // First line specifies the mode; it doesn't affect what we do so we can just skip it. + + for s.Scan() { + line := s.Text() + + cols := strings.Fields(line) + key := strings.Split(cols[0], ":")[0] + + numStmts, err := strconv.Atoi(cols[1]) + if err != nil { + panic(err) + } + numTimesCovered, err := strconv.Atoi(cols[2]) + if err != nil { + panic(err) + } + + if val, ok := agg[key]; ok { + val.NumStmts += numStmts + if numTimesCovered > 0 { + val.NumCoveredStmts += numStmts + } + } else { + numCoveredStmts := 0 + if numTimesCovered > 0 { + numCoveredStmts = numStmts + } + + agg[key] = &aggregation{ + NumStmts: numStmts, + NumCoveredStmts: numCoveredStmts, + } + } + } + + keys := make([]string, 0, len(agg)) + for k := range agg { + keys = append(keys, k) + } + sort.Strings(keys) + + fmt.Println("Go coverage report:") + fmt.Println("
") + fmt.Println("Click to expand.") + fmt.Println("") + fmt.Println("| File | % |") + fmt.Println("| ---- | - |") + + totalStmts := 0 + totalCoveredStmts := 0 + + for _, k := range keys { + a := agg[k] + fmt.Printf("| %s | %s |\n", k, a.CoveragePct()) + + totalStmts += a.NumStmts + totalCoveredStmts += a.NumCoveredStmts + } + + fmt.Printf("| total | %.1f%% |\n", (float64(totalCoveredStmts)/float64(totalStmts))*100) + fmt.Println("
") +} diff --git a/.drone/cmd/ghcomment/main.go b/.drone/cmd/ghcomment/main.go new file mode 100644 index 000000000..61e96e86b --- /dev/null +++ b/.drone/cmd/ghcomment/main.go @@ -0,0 +1,97 @@ +package main + +import ( + "context" + "flag" + "io/ioutil" + "os" + "strconv" + "strings" + + "github.com/shurcooL/githubv4" + + "github.com/go-graphite/carbonapi/drone/pkg/github" +) + +const CommenterLogin = "grafanabot" + +func main() { + grafanabotPat := getRequiredEnv("GRAFANABOT_PAT") + + repoOwner := getRequiredEnv("DRONE_REPO_OWNER") + repoName := getRequiredEnv("DRONE_REPO_NAME") + + pullRequest, err := strconv.Atoi(getRequiredEnv("DRONE_PULL_REQUEST")) + if err != nil { + panic(err) + } + + commentTypeIdentifier := flag.String("id", "", "String that identifies the comment type being submitted") + commentBodyFilename := flag.String("bodyfile", "", "A file containing the comment body") + flag.Parse() + + if *commentTypeIdentifier == "" { + panic("Required argument: -i") + } + if *commentBodyFilename == "" { + panic("Required argument: -b") + } + + api := github.NewAPI(context.Background(), grafanabotPat) + + err = minimizeOutdatedComments(api, repoOwner, repoName, pullRequest, *commentTypeIdentifier) + if err != nil { + panic(err) + } + + commentBody, err := ioutil.ReadFile(*commentBodyFilename) + if err != nil { + panic(err) + } + + err = addComment(api, repoOwner, repoName, pullRequest, string(commentBody)) + if err != nil { + panic(err) + } +} + +func getRequiredEnv(k string) string { + v, p := os.LookupEnv(k) + if !p { + panic("Missing required env var: " + k) + } + + return v +} + +func minimizeOutdatedComments(api *github.API, repoOwner string, repoName string, pullRequestNo int, commentTypeIdentifier string) error { + prComments, err := api.ListPullRequestComments(repoOwner, repoName, pullRequestNo) + if err != nil { + return err + } + + for _, comment := range prComments { + if comment.Author.Login == CommenterLogin && strings.Contains(comment.Body, commentTypeIdentifier) && !comment.IsMinimized { + err := api.MinimizeComment(comment.ID, githubv4.ReportedContentClassifiersOutdated) + if err != nil { + return err + } + } + } + + return nil +} + +func addComment(api *github.API, repoOwner string, repoName string, pullRequestNo int, commentBody string) error { + pullRequestNodeID, err := api.GetPullRequestNodeID(repoOwner, repoName, pullRequestNo) + if err != nil { + return err + } + + _, err = api.AddComment(pullRequestNodeID, commentBody) + if err != nil { + return err + } + + return nil +} diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet new file mode 100644 index 000000000..2acd6f39b --- /dev/null +++ b/.drone/drone.jsonnet @@ -0,0 +1,93 @@ +local drone = import 'lib/drone/drone.libsonnet'; +local images = import 'lib/drone/images.libsonnet'; +local triggers = import 'lib/drone/triggers.libsonnet'; +local vault = import 'lib/vault/vault.libsonnet'; + +local pipeline = drone.pipeline; +local step = drone.step; +local withInlineStep = drone.withInlineStep; +local withStep = drone.withStep; +local withSteps = drone.withSteps; + +local imagePullSecrets = { image_pull_secrets: ['dockerconfigjson'] }; + +local commentCoverageLintReport = { + step: step('coverage + lint', $.commands, image=$.image, environment=$.environment), + commands: [ + // Build drone utilities. + 'scripts/build-drone-utilities.sh', + // Generate the raw coverage report. + 'go test -coverprofile=coverage.out ./...', + // Process the raw coverage report. + '.drone/coverage > coverage_report.out', + // Generate the lint report. + 'scripts/generate-lint-report.sh', + // Combine the reports. + 'cat coverage_report.out > report.out', + 'echo "" >> report.out', + 'cat lint.out >> report.out', + // Submit the comment to GitHub. + '.drone/ghcomment -id "Go coverage report:" -bodyfile report.out', + ], + environment: { + GRAFANABOT_PAT: { from_secret: 'gh_token' }, + }, + image: images._images.goLint, +}; + +local buildAndPushImages = { + // step builds the pipeline step to build and push a docker image + step(app): step( + '%s: build and push' % app, + [], + image=buildAndPushImages.pluginName, + settings=buildAndPushImages.settings(app), + ), + + pluginName: 'plugins/gcr', + + // settings generates the CI Pipeline step settings + settings(app): { + repo: $._repo(app), + registry: $._registry, + dockerfile: './Dockerfile', + json_key: { from_secret: 'gcr_admin' }, + mirror: 'https://mirror.gcr.io', + build_args: ['cmd=' + app], + }, + + // image generates the image for the given app + image(app): $._registry + '/' + $._repo(app), + + _repo(app):: 'kubernetes-dev/' + app, + _registry:: 'us.gcr.io', +}; + +local runTests = { + step: step('run tests', $.commands, image=$.image), + commands: [ + 'make test' + ], + image: images._images.testRunner, + settings: { + + } +}; + +[ + pipeline('test') + + withStep(runTests.step) + + imagePullSecrets + + triggers.pr + + triggers.main, + + pipeline('coverageLintReport') + + withStep(commentCoverageLintReport.step) + + triggers.pr, +] ++ [ + vault.secret('dockerconfigjson', 'secret/data/common/gcr', '.dockerconfigjson'), + vault.secret('gh_token', 'infra/data/ci/github/grafanabot', 'pat'), + vault.secret('gcr_admin', 'infra/data/ci/gcr-admin', 'service-account'), + vault.secret('argo_token', 'infra/data/ci/argo-workflows/trigger-service-account', 'token'), +] \ No newline at end of file diff --git a/.drone/drone.yml b/.drone/drone.yml new file mode 100644 index 000000000..1aa27e8ae --- /dev/null +++ b/.drone/drone.yml @@ -0,0 +1,79 @@ +--- +depends_on: null +image_pull_secrets: +- dockerconfigjson +kind: pipeline +name: test +steps: +- commands: + - make test + depends_on: [] + entrypoint: null + environment: {} + image: us.gcr.io/kubernetes-dev/carbonapi/test-runner:latest + name: run tests + settings: {} +trigger: + branch: + - main + event: + include: + - pull_request + - push +type: docker +--- +depends_on: null +kind: pipeline +name: coverageLintReport +steps: +- commands: + - scripts/build-drone-utilities.sh + - go test -coverprofile=coverage.out ./... + - .drone/coverage > coverage_report.out + - scripts/generate-lint-report.sh + - cat coverage_report.out > report.out + - echo "" >> report.out + - cat lint.out >> report.out + - .drone/ghcomment -id "Go coverage report:" -bodyfile report.out + depends_on: [] + entrypoint: null + environment: + GRAFANABOT_PAT: + from_secret: gh_token + image: golangci/golangci-lint:v1.45 + name: coverage + lint + settings: {} +trigger: + event: + include: + - pull_request +type: docker +--- +get: + name: .dockerconfigjson + path: secret/data/common/gcr +kind: secret +name: dockerconfigjson +--- +get: + name: pat + path: infra/data/ci/github/grafanabot +kind: secret +name: gh_token +--- +get: + name: service-account + path: infra/data/ci/gcr-admin +kind: secret +name: gcr_admin +--- +get: + name: token + path: infra/data/ci/argo-workflows/trigger-service-account +kind: secret +name: argo_token +--- +kind: signature +hmac: 786972e7020e02e75b297bf8f7a67669cf11bd5bf02f61ac62ec63d0fca2a6c1 + +... diff --git a/.drone/go.mod b/.drone/go.mod new file mode 100644 index 000000000..8d8bf331e --- /dev/null +++ b/.drone/go.mod @@ -0,0 +1,16 @@ +module github.com/go-graphite/carbonapi/drone + +go 1.18 + +require ( + github.com/shurcooL/githubv4 v0.0.0-20220520033151-0b4e3294ff00 + golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 +) + +require ( + github.com/golang/protobuf v1.5.2 // indirect + github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29 // indirect + golang.org/x/net v0.0.0-20220706163947-c90051bbdb60 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.0 // indirect +) diff --git a/.drone/go.sum b/.drone/go.sum new file mode 100644 index 000000000..1b386f507 --- /dev/null +++ b/.drone/go.sum @@ -0,0 +1,27 @@ +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/shurcooL/githubv4 v0.0.0-20220520033151-0b4e3294ff00 h1:fiFvD4lT0aWjuuAb64LlZ/67v87m+Kc9Qsu5cMFNK0w= +github.com/shurcooL/githubv4 v0.0.0-20220520033151-0b4e3294ff00/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= +github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29 h1:B1PEwpArrNp4dkQrfxh/abbBAOZBVp0ds+fBEOUOqOc= +github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20220706163947-c90051bbdb60 h1:8NSylCMxLW4JvserAndSgFL7aPli6A68yf0bYFTcWCM= +golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 h1:VnGaRqoLmqZH/3TMLJwYCEWkR4j1nuIU1U9TvbqsDUw= +golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/.drone/lib/drone/drone.libsonnet b/.drone/lib/drone/drone.libsonnet new file mode 100644 index 000000000..9f3a68787 --- /dev/null +++ b/.drone/lib/drone/drone.libsonnet @@ -0,0 +1,46 @@ +local images = import 'images.libsonnet'; +{ + step(name, + commands, + image=images._images.go, + settings={}, + environment={}, + entrypoint=null, + depends_on=[], + dir=null):: { + name: name, + entrypoint: entrypoint, + commands: if (dir == null || dir == '') then commands else ['cd %s' % dir] + commands, + image: image, + settings: settings, + environment: environment, + depends_on: depends_on, + }, + + withStep(step):: { + steps+: [step], + }, + + withSteps(steps):: { + steps+: steps, + }, + + + withInlineStep(name, + commands, + image=images._images.go, + settings={}, + environment={}, + entrypoint=null, + depends_on=[], + dir=null):: + $.withStep($.step(name, commands, image, settings, environment, entrypoint, depends_on, dir)), + + pipeline(name, steps=[], depends_on=null):: { + kind: 'pipeline', + type: 'docker', + name: name, + steps: steps, + depends_on: depends_on, + }, +} \ No newline at end of file diff --git a/.drone/lib/drone/images.libsonnet b/.drone/lib/drone/images.libsonnet new file mode 100644 index 000000000..df032b117 --- /dev/null +++ b/.drone/lib/drone/images.libsonnet @@ -0,0 +1,9 @@ +{ + _images+:: { + argoCli: 'us.gcr.io/kubernetes-dev/drone/plugins/argo-cli', + go: 'golang:1.17', + goLint: 'golangci/golangci-lint:v1.45', + dind: 'docker:dind', + testRunner: 'us.gcr.io/kubernetes-dev/carbonapi/test-runner:latest', + }, +} \ No newline at end of file diff --git a/.drone/lib/drone/triggers.libsonnet b/.drone/lib/drone/triggers.libsonnet new file mode 100644 index 000000000..aa0146905 --- /dev/null +++ b/.drone/lib/drone/triggers.libsonnet @@ -0,0 +1,25 @@ +{ + main:: { + trigger+: { + branch+: ['main'], + event+: { + include+: ['push'], + }, + }, + }, + pr:: { + trigger+: { + event+: { + include+: ['pull_request'], + }, + }, + }, + // excluding paths disables runs that contain changes to ONLY these files + excludeModifiedPaths(paths):: { + trigger+: { + paths+: { + exclude+: paths, + }, + }, + }, +} \ No newline at end of file diff --git a/.drone/lib/vault/vault.libsonnet b/.drone/lib/vault/vault.libsonnet new file mode 100644 index 000000000..dfaf4f762 --- /dev/null +++ b/.drone/lib/vault/vault.libsonnet @@ -0,0 +1,10 @@ +{ + secret(name, vault_path, key):: { + kind: 'secret', + name: name, + get: { + path: vault_path, + name: key, + }, + }, +} \ No newline at end of file diff --git a/.drone/pkg/github/github.go b/.drone/pkg/github/github.go new file mode 100644 index 000000000..0b0ae99fa --- /dev/null +++ b/.drone/pkg/github/github.go @@ -0,0 +1,139 @@ +package github + +import ( + "context" + "github.com/shurcooL/githubv4" + "golang.org/x/oauth2" +) + +type API struct { + ctx *context.Context + githubClient *githubv4.Client +} + +func NewAPI(ctx context.Context, pat string) *API { + src := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: pat}) + httpClient := oauth2.NewClient(ctx, src) + githubClient := githubv4.NewClient(httpClient) + + return &API{ + ctx: &ctx, + githubClient: githubClient, + } +} + +type PullRequestComment struct { + ID githubv4.ID + Body string + IsMinimized bool + Author struct { + Login string + } +} + +func (a *API) ListPullRequestComments(repoOwner string, repoName string, pullRequestNo int) ([]PullRequestComment, error) { + var q struct { + Repository struct { + PullRequest struct { + Comments struct { + Nodes []PullRequestComment + PageInfo struct { + EndCursor githubv4.String + HasNextPage bool + } + } `graphql:"comments(first: 100, after: $commentsCursor)"` + } `graphql:"pullRequest(number: $pr)"` + } `graphql:"repository(owner: $owner, name: $name)"` + } + + variables := map[string]interface{}{ + "owner": githubv4.String(repoOwner), + "name": githubv4.String(repoName), + "pr": githubv4.Int(pullRequestNo), + "commentsCursor": (*githubv4.String)(nil), + } + + var allComments []PullRequestComment + for { + err := a.githubClient.Query(*a.ctx, &q, variables) + if err != nil { + return nil, err + } + + allComments = append(allComments, q.Repository.PullRequest.Comments.Nodes...) + + if !q.Repository.PullRequest.Comments.PageInfo.HasNextPage { + break + } + variables["commentsCursor"] = githubv4.NewString(q.Repository.PullRequest.Comments.PageInfo.EndCursor) + } + + return allComments, nil +} + +func (a *API) MinimizeComment(commentNodeID githubv4.ID, classifier githubv4.ReportedContentClassifiers) error { + var m struct { + MinimizeComment struct { + MinimizedComment struct { + IsMinimized bool + } + } `graphql:"minimizeComment(input: $input)"` + } + input := githubv4.MinimizeCommentInput{ + SubjectID: commentNodeID, + Classifier: classifier, + } + + err := a.githubClient.Mutate(*a.ctx, &m, input, nil) + if err != nil { + return err + } + + return nil +} + +func (a *API) GetPullRequestNodeID(repoOwner string, repoName string, pullRequestNo int) (githubv4.ID, error) { + var q struct { + Repository struct { + PullRequest struct { + ID githubv4.ID + } `graphql:"pullRequest(number: $pr)"` + } `graphql:"repository(owner: $owner, name: $name)"` + } + + variables := map[string]interface{}{ + "owner": githubv4.String(repoOwner), + "name": githubv4.String(repoName), + "pr": githubv4.Int(pullRequestNo), + } + + err := a.githubClient.Query(*a.ctx, &q, variables) + if err != nil { + return "", err + } + + return q.Repository.PullRequest.ID, nil +} + +func (a *API) AddComment(pullRequestNodeID githubv4.ID, commentBody string) (githubv4.ID, error) { + var m struct { + AddComment struct { + CommentEdge struct { + Node struct { + ID githubv4.ID + } + } + } `graphql:"addComment(input: $input)"` + } + input := githubv4.AddCommentInput{ + SubjectID: pullRequestNodeID, + Body: githubv4.String(commentBody), + } + + err := a.githubClient.Mutate(*a.ctx, &m, input, nil) + if err != nil { + return "", err + } + + return m.AddComment.CommentEdge.Node.ID, nil +} diff --git a/.github/workflows/sync-upstream.yaml b/.github/workflows/sync-upstream.yaml new file mode 100644 index 000000000..24014814f --- /dev/null +++ b/.github/workflows/sync-upstream.yaml @@ -0,0 +1,44 @@ +name: Sync fork with upstream + +on: + schedule: + - cron: "12 7 * * 1" + # scheduled at 07:12 every Monday + workflow_dispatch: + +jobs: + sync_with_upstream: + runs-on: ubuntu-latest + name: Sync HEAD with upstream latest + + steps: + # Step 1: run a standard checkout action, provided by github + - name: Checkout HEAD + uses: actions/checkout@v2 + with: + ref: main + + # Step 2: run this sync action - specify the upstream repo, upstream branch to sync with, and target sync branch + - name: Pull upstream changes + id: sync + uses: aormsby/Fork-Sync-With-Upstream-action@v3.2 + with: + target_sync_branch: main + target_repo_token: ${{ secrets.GITHUB_TOKEN }} + upstream_sync_repo: go-graphite/carbonapi + upstream_sync_branch: main + + # Set test_mode true to run tests instead of the true action!! + test_mode: true + + # Step 3: Display a sample message based on the sync output var 'has_new_commits' + - name: New commits found + if: steps.sync.outputs.has_new_commits == 'true' + run: echo "New commits were found to sync." + + - name: No new commits + if: steps.sync.outputs.has_new_commits == 'false' + run: echo "There were no new commits." + + - name: Show value of 'has_new_commits' + run: echo ${{ steps.sync.outputs.has_new_commits }} diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index e4c3082d4..805d9325b 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -149,7 +149,7 @@ limit: default value mismatch: got "INF", should be "Infinity" | | removeAbovePercentile | n: type mismatch: got integer, should be float | | removeAboveValue | n: type mismatch: got integer, should be float | | removeBelowPercentile | n: type mismatch: got integer, should be float | -| removeBelowValue | n: type mismatch: got integer, should be float | +| removeBelowValue | n: type mismatch: got integer, sho/usr/bin/nvidia-dockeruld be float | | round | precision: default value mismatch: got (empty), should be 0 | | scaleToSeconds | seconds: type mismatch: got integer, should be float | | smartSummarize | func: different amount of parameters, `[current rangeOf]` are missing @@ -162,6 +162,7 @@ reverse: default value mismatch: got (empty), should be false | | timeShift | parameter not supported: alignDst | | timeSlice | endSliceAt: type mismatch: got interval, should be date | +## Supported functions ## Supported functions | Function | Carbonapi-only | | :-------------|:--------------------------------------------------------- | @@ -293,6 +294,7 @@ reverse: default value mismatch: got (empty), should be false | | substr(seriesList, start=0, stop=0) | no | | sum(*seriesLists) | no | | sumSeries(*seriesLists) | no | +| sumSeriesLists(seriesListFirstPos, seriesListSecondPos) | no | | sumSeriesWithWildcards(seriesList, *position) | no | | summarize(seriesList, intervalString, func='sum', alignToFrom=False) | no | | threshold(value, label=None, color=None) | no | @@ -301,6 +303,8 @@ reverse: default value mismatch: got (empty), should be false | | timeShift(seriesList, timeShift, resetEnd=True, alignDST=False) | no | | timeSlice(seriesList, startSliceAt, endSliceAt='now') | no | | timeStack(seriesList, timeShiftUnit='1d', timeShiftStart=0, timeShiftEnd=7) | no | +| toLowerCase(seriesList) | no | +| toUpperCase(seriesList, *pos) | no | | transformNull(seriesList, default=0, referenceSeries=None) | no | | unique(*seriesLists) | no | | useSeriesAbove(seriesList, value, search, replace) | no | diff --git a/expr/consolidations/consolidations.go b/expr/consolidations/consolidations.go index d3ba4151f..dbcd54a32 100644 --- a/expr/consolidations/consolidations.go +++ b/expr/consolidations/consolidations.go @@ -1,19 +1,16 @@ package consolidations import ( + "github.com/go-graphite/carbonapi/pkg/errors" "math" "regexp" "strconv" "strings" - "github.com/ansel1/merry" - "github.com/wangjohn/quickselect" "gonum.org/v1/gonum/mat" ) -var ErrInvalidConsolidationFunc = merry.New("Invalid Consolidation Function") - // ConsolidationToFunc contains a map of graphite-compatible consolidation functions definitions to actual functions that can do aggregation // TODO(civil): take into account xFilesFactor var ConsolidationToFunc = map[string]func([]float64) float64{ @@ -49,7 +46,7 @@ func CheckValidConsolidationFunc(functionName string) error { return nil } } - return ErrInvalidConsolidationFunc.WithMessage("invalid consolidation " + functionName) + return errors.ErrUnsupportedConsolidationFunction{Func: functionName} } // AvgValue returns average of list of values diff --git a/expr/consolidations/consolidations_test.go b/expr/consolidations/consolidations_test.go index 78998883f..64630042e 100644 --- a/expr/consolidations/consolidations_test.go +++ b/expr/consolidations/consolidations_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/ansel1/merry" + "github.com/go-graphite/carbonapi/pkg/errors" ) func TestSummarizeValues(t *testing.T) { @@ -164,7 +165,7 @@ func TestCheckValidConsolidationFunc(t *testing.T) { }, { name: "test", - expectedResult: ErrInvalidConsolidationFunc, + expectedResult: errors.ErrUnsupportedConsolidationFunction{Func: "test"}, }, } diff --git a/expr/expr.go b/expr/expr.go index e43a49a78..c393690f1 100644 --- a/expr/expr.go +++ b/expr/expr.go @@ -2,6 +2,7 @@ package expr import ( "context" + "github.com/go-graphite/carbonapi/pkg/errors" utilctx "github.com/go-graphite/carbonapi/util/ctx" @@ -149,8 +150,7 @@ func EvalExpr(ctx context.Context, e parser.Expr, from, until int64, values map[ // all functions have arguments -- check we do too if e.ArgsLen() == 0 { - err := merry.WithMessagef(parser.ErrMissingArgument, "target=%s: %s", e.Target(), parser.ErrMissingArgument) - return nil, merry.WithHTTPCode(err, 400) + return nil, errors.ErrMissingArgument{Target: e.Target()} } metadata.FunctionMD.RLock() @@ -170,6 +170,8 @@ func EvalExpr(ctx context.Context, e parser.Expr, from, until int64, values map[ parser.ErrMissingArgument, parser.ErrMissingTimeseries, parser.ErrUnknownTimeUnits, + errors.ErrUnsupportedConsolidationFunction{}, + errors.ErrInvalidArgument{}, ) { err = merry.WithHTTPCode(err, 400) } @@ -177,7 +179,7 @@ func EvalExpr(ctx context.Context, e parser.Expr, from, until int64, values map[ return v, err } - return nil, merry.WithHTTPCode(helper.ErrUnknownFunction(e.Target()), 400) + return nil, errors.ErrUnknownFunction(e.Target()) } // RewriteExpr expands targets that use applyByNode into a new list of targets. diff --git a/expr/functions/aggregate/function.go b/expr/functions/aggregate/function.go index e2e76354b..e8bb4b9f9 100644 --- a/expr/functions/aggregate/function.go +++ b/expr/functions/aggregate/function.go @@ -2,7 +2,7 @@ package aggregate import ( "context" - "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "strings" "github.com/go-graphite/carbonapi/expr/consolidations" @@ -77,7 +77,7 @@ func (f *aggregate) Do(ctx context.Context, e parser.Expr, from, until int64, va aggFunc, ok := consolidations.ConsolidationToFunc[callback] if !ok { - return nil, fmt.Errorf("unsupported consolidation function %s", callback) + return nil, errors.ErrUnsupportedConsolidationFunction{Func: callback} } target := callback + "Series" diff --git a/expr/functions/aggregateLine/function.go b/expr/functions/aggregateLine/function.go index 17f5a5562..e92c7caff 100644 --- a/expr/functions/aggregateLine/function.go +++ b/expr/functions/aggregateLine/function.go @@ -2,7 +2,7 @@ package aggregateLine import ( "context" - "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "strconv" @@ -59,7 +59,7 @@ func (f *aggregateLine) Do(ctx context.Context, e parser.Expr, from, until int64 aggFunc, ok := consolidations.ConsolidationToFunc[callback] if !ok { - return nil, fmt.Errorf("unsupported consolidation function %s", callback) + return nil, errors.ErrUnsupportedConsolidationFunction{Func: callback} } results := make([]*types.MetricData, len(args)) diff --git a/expr/functions/aggregateSeriesLists/function.go b/expr/functions/aggregateSeriesLists/function.go index 43a6bb055..d28cf355b 100644 --- a/expr/functions/aggregateSeriesLists/function.go +++ b/expr/functions/aggregateSeriesLists/function.go @@ -2,7 +2,7 @@ package aggregateSeriesLists import ( "context" - "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "github.com/go-graphite/carbonapi/expr/consolidations" "github.com/go-graphite/carbonapi/expr/helper" @@ -39,7 +39,7 @@ func (f *aggregateSeriesLists) Do(ctx context.Context, e parser.Expr, from, unti } if len(seriesList1) != len(seriesList2) { - return nil, fmt.Errorf("seriesListFirstPos and seriesListSecondPos must have equal length") + return nil, errors.ErrBadData{Target: e.Target(), Msg: "seriesListFirstPos and seriesListSecondPos must have equal length"} } else if len(seriesList1) == 0 { return make([]*types.MetricData, 0, 0), nil } @@ -50,7 +50,7 @@ func (f *aggregateSeriesLists) Do(ctx context.Context, e parser.Expr, from, unti } aggFunc, ok := consolidations.ConsolidationToFunc[aggFuncStr] if !ok { - return nil, fmt.Errorf("unsupported consolidation function %s", aggFuncStr) + return nil, errors.ErrUnsupportedConsolidationFunction{Func: aggFuncStr} } xFilesFactor, err := e.GetFloatArgDefault(3, float64(seriesList1[0].XFilesFactor)) diff --git a/expr/functions/aggregateWithWildcards/function.go b/expr/functions/aggregateWithWildcards/function.go index 4fbf61b6a..ef8ceed9c 100644 --- a/expr/functions/aggregateWithWildcards/function.go +++ b/expr/functions/aggregateWithWildcards/function.go @@ -3,6 +3,7 @@ package aggregateWithWildcards import ( "context" "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "strings" "github.com/go-graphite/carbonapi/expr/consolidations" @@ -55,7 +56,7 @@ func (f *aggregateWithWildcards) Do(ctx context.Context, e parser.Expr, from, un aggFunc, ok := consolidations.ConsolidationToFunc[callback] if !ok { - return nil, fmt.Errorf("unsupported consolidation function %s", callback) + return nil, errors.ErrUnsupportedConsolidationFunction{Func: callback} } target := fmt.Sprintf("%sSeries", callback) e.SetTarget(target) diff --git a/expr/functions/asPercent/function.go b/expr/functions/asPercent/function.go index ec1448386..8cac25568 100644 --- a/expr/functions/asPercent/function.go +++ b/expr/functions/asPercent/function.go @@ -2,13 +2,13 @@ package asPercent import ( "context" - "errors" "math" "sort" "github.com/go-graphite/carbonapi/expr/helper" "github.com/go-graphite/carbonapi/expr/interfaces" "github.com/go-graphite/carbonapi/expr/types" + "github.com/go-graphite/carbonapi/pkg/errors" "github.com/go-graphite/carbonapi/pkg/parser" ) @@ -433,7 +433,7 @@ func (f *asPercent) Do(ctx context.Context, e parser.Expr, from, until int64, va } } - return nil, errors.New("total must be either a constant or a series") + return nil, errors.ErrInvalidArgument{Target: e.Target(), Msg: "total must be either a constant or a series"} } // Description is auto-generated description, based on output of https://github.com/graphite-project/graphite-web diff --git a/expr/functions/cactiStyle/function.go b/expr/functions/cactiStyle/function.go index 49d487042..2b2a80993 100644 --- a/expr/functions/cactiStyle/function.go +++ b/expr/functions/cactiStyle/function.go @@ -3,6 +3,7 @@ package cactiStyle import ( "context" "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "strings" @@ -92,7 +93,7 @@ func (f *cactiStyle) Do(ctx context.Context, e parser.Expr, from, until int64, v current = fmt.Sprintf("%.0f", currentVal) } else { - return nil, fmt.Errorf("%s is not supported for system", system) + return nil, errors.ErrBadData{Target: e.Target(), Msg: system + "is not supported for system"} } // Append the unit if specified diff --git a/expr/functions/cairo/png/cairo.go b/expr/functions/cairo/png/cairo.go index 4ebcd9239..97194f22f 100644 --- a/expr/functions/cairo/png/cairo.go +++ b/expr/functions/cairo/png/cairo.go @@ -19,6 +19,7 @@ import ( "github.com/go-graphite/carbonapi/expr/helper" "github.com/go-graphite/carbonapi/expr/types" + "github.com/go-graphite/carbonapi/pkg/errors" "github.com/go-graphite/carbonapi/pkg/parser" pb "github.com/go-graphite/protocol/carbonapi_v3_pb" @@ -879,7 +880,7 @@ func EvalExprGraph(ctx context.Context, e parser.Expr, from, until int64, values } - return nil, helper.ErrUnknownFunction(e.Target()) + return nil, errors.ErrUnknownFunction(e.Target()) } func MarshalSVG(params PictureParams, results []*types.MetricData) []byte { diff --git a/expr/functions/divideSeries/function.go b/expr/functions/divideSeries/function.go index 0c483c08a..33708c6bb 100644 --- a/expr/functions/divideSeries/function.go +++ b/expr/functions/divideSeries/function.go @@ -2,8 +2,8 @@ package divideSeries import ( "context" - "errors" "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "github.com/go-graphite/carbonapi/expr/helper" @@ -33,7 +33,7 @@ func New(configFile string) []interfaces.FunctionMetadata { // divideSeries(dividendSeriesList, divisorSeriesList) func (f *divideSeries) Do(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) { if e.ArgsLen() < 1 { - return nil, parser.ErrMissingTimeseries + return nil, errors.ErrMissingTimeseries{Target: e.Target()} } firstArg, err := helper.GetSeriesArg(ctx, e.Arg(0), from, until, values) @@ -69,7 +69,7 @@ func (f *divideSeries) Do(ctx context.Context, e parser.Expr, from, until int64, } if len(denominators) > 1 { - return nil, types.ErrWildcardNotAllowed + return nil, errors.ErrWildcardNotAllowed{Target: e.Target(), Arg: e.Arg(1).ToString()} } denominator = denominators[0] @@ -77,7 +77,7 @@ func (f *divideSeries) Do(ctx context.Context, e parser.Expr, from, until int64, numerators = append(numerators, firstArg[0]) denominator = firstArg[1] } else { - return nil, errors.New("must be called with 2 series or a wildcard that matches exactly 2 series") + return nil, errors.ErrBadData{Target: e.Target(), Msg: "must be called with 2 series or a wildcard that matches exactly 2 series"} } for _, numerator := range numerators { diff --git a/expr/functions/exponentialMovingAverage/function.go b/expr/functions/exponentialMovingAverage/function.go index 97e6cae3f..3eafb5355 100644 --- a/expr/functions/exponentialMovingAverage/function.go +++ b/expr/functions/exponentialMovingAverage/function.go @@ -3,6 +3,7 @@ package exponentialMovingAverage import ( "context" "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "strconv" @@ -38,7 +39,7 @@ func (f *exponentialMovingAverage) Do(ctx context.Context, e parser.Expr, from, var argstr string if len(e.Args()) < 2 { - return nil, parser.ErrMissingArgument + return nil, errors.ErrMissingArgument{Target: e.Target()} } switch e.Args()[1].Type() { @@ -56,7 +57,7 @@ func (f *exponentialMovingAverage) Do(ctx context.Context, e parser.Expr, from, argstr = fmt.Sprintf("%q", e.Args()[1].StringValue()) n = int(n32) default: - err = parser.ErrBadType + err = errors.ErrBadType{Arg: e.Arg(1).ToString(), Exp: parser.TypeToString(parser.EtConst) + " or " + parser.TypeToString(parser.EtString), Got: parser.TypeToString(e.Args()[1].Type())} } if err != nil { return nil, err @@ -65,6 +66,10 @@ func (f *exponentialMovingAverage) Do(ctx context.Context, e parser.Expr, from, var results []*types.MetricData windowSize := n + if windowSize < 1 { + return nil, fmt.Errorf("invalid window size %d", windowSize) + } + start := from arg, err := helper.GetSeriesArg(ctx, e.Args()[0], start, until, values) @@ -83,20 +88,22 @@ func (f *exponentialMovingAverage) Do(ctx context.Context, e parser.Expr, from, var vals []float64 - if windowSize < 1 && windowSize > len(a.Values) { - return nil, fmt.Errorf("invalid window size %d", windowSize) - } - ema := consolidations.AggMean(a.Values[:windowSize]) + if windowSize > len(a.Values) { + mean := consolidations.AggMean(a.Values) + vals = append(vals, helper.SafeRound(mean, 6)) + } else { + ema := consolidations.AggMean(a.Values[:windowSize]) - vals = append(vals, helper.SafeRound(ema, 6)) - for _, v := range a.Values[windowSize:] { - if math.IsNaN(v) { - vals = append(vals, math.NaN()) - continue - } - ema = constant*v + (1-constant)*ema vals = append(vals, helper.SafeRound(ema, 6)) + for _, v := range a.Values[windowSize:] { + if math.IsNaN(v) { + vals = append(vals, math.NaN()) + continue + } + ema = constant*v + (1-constant)*ema + vals = append(vals, helper.SafeRound(ema, 6)) + } } r.Tags[e.Target()] = fmt.Sprintf("%d", windowSize) diff --git a/expr/functions/exponentialMovingAverage/function_test.go b/expr/functions/exponentialMovingAverage/function_test.go index 23f335163..06e1eeeaf 100644 --- a/expr/functions/exponentialMovingAverage/function_test.go +++ b/expr/functions/exponentialMovingAverage/function_test.go @@ -33,6 +33,17 @@ func TestExponentialMovingAverage(t *testing.T) { types.MakeMetricData("exponentialMovingAverage(metric1,3)", []float64{4, 6, 9, 11.5, 13.75, 15.875, 17.9375}, 1, 0), }, }, + { + // if the window is larger than the length of the values, the result should just be the average. + // this matches graphiteweb's behavior + "exponentialMovingAverage(metric1,100)", + map[parser.MetricRequest][]*types.MetricData{ + {"metric1", 0, 1}: {types.MakeMetricData("metric1", []float64{1, 2, 3}, 1, startTime)}, + }, + []*types.MetricData{ + types.MakeMetricData("exponentialMovingAverage(metric1,100)", []float64{2}, 1, 0), + }, + }, } for _, tt := range tests { diff --git a/expr/functions/fallbackSeries/function.go b/expr/functions/fallbackSeries/function.go index 2a33bccec..180bdd6f0 100644 --- a/expr/functions/fallbackSeries/function.go +++ b/expr/functions/fallbackSeries/function.go @@ -6,6 +6,7 @@ import ( "github.com/go-graphite/carbonapi/expr/helper" "github.com/go-graphite/carbonapi/expr/interfaces" "github.com/go-graphite/carbonapi/expr/types" + "github.com/go-graphite/carbonapi/pkg/errors" "github.com/go-graphite/carbonapi/pkg/parser" ) @@ -34,7 +35,7 @@ func (f *fallbackSeries) Do(ctx context.Context, e parser.Expr, from, until int6 If the wildcard does not match any series, draws the fallback metric. */ if e.ArgsLen() < 2 { - return nil, parser.ErrMissingTimeseries + return nil, errors.ErrMissingTimeseries{Target: e.Target()} } seriesList, err := helper.GetSeriesArg(ctx, e.Arg(0), from, until, values) diff --git a/expr/functions/fallbackSeries/function_test.go b/expr/functions/fallbackSeries/function_test.go index fcd9c3ba2..57b76b7fe 100644 --- a/expr/functions/fallbackSeries/function_test.go +++ b/expr/functions/fallbackSeries/function_test.go @@ -1,6 +1,7 @@ package fallbackSeries import ( + "github.com/go-graphite/carbonapi/pkg/errors" "testing" "time" @@ -56,7 +57,7 @@ func TestFallbackSeries(t *testing.T) { }, }, { - "fallbackSeries(metric1,metrc2)", + "fallbackSeries(metric1,metric2)", map[parser.MetricRequest][]*types.MetricData{ {"metric1", 0, 1}: {types.MakeMetricData("metric1", []float64{0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9}, 1, now32)}, }, @@ -89,7 +90,7 @@ func TestErrorMissingTimeSeriesFunction(t *testing.T) { }, }, nil, - parser.ErrMissingTimeseries, + errors.ErrMissingTimeseries{Target: "fallbackSeries"}, }, } diff --git a/expr/functions/filter/function.go b/expr/functions/filter/function.go index cb7f98eae..9cd9b8658 100644 --- a/expr/functions/filter/function.go +++ b/expr/functions/filter/function.go @@ -3,6 +3,7 @@ package filter import ( "context" "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "github.com/go-graphite/carbonapi/expr/consolidations" "github.com/go-graphite/carbonapi/expr/helper" @@ -65,7 +66,7 @@ func (f *filterSeries) Do(ctx context.Context, e parser.Expr, from, until int64, aggFunc, ok := consolidations.ConsolidationToFunc[callback] if !ok { - return nil, fmt.Errorf("unsupported consolidation function %s", callback) + return nil, errors.ErrUnsupportedConsolidationFunction{Func: callback} } results := make([]*types.MetricData, 0, len(args)) diff --git a/expr/functions/glue.go b/expr/functions/glue.go index 090bcbb67..61bb80027 100644 --- a/expr/functions/glue.go +++ b/expr/functions/glue.go @@ -110,6 +110,8 @@ import ( "github.com/go-graphite/carbonapi/expr/functions/timeShiftByMetric" "github.com/go-graphite/carbonapi/expr/functions/timeSlice" "github.com/go-graphite/carbonapi/expr/functions/timeStack" + "github.com/go-graphite/carbonapi/expr/functions/toLowerCase" + "github.com/go-graphite/carbonapi/expr/functions/toUpperCase" "github.com/go-graphite/carbonapi/expr/functions/transformNull" "github.com/go-graphite/carbonapi/expr/functions/tukey" "github.com/go-graphite/carbonapi/expr/functions/unique" @@ -234,6 +236,8 @@ func New(configs map[string]string) { {name: "timeShiftByMetric", filename: "timeShiftByMetric", order: timeShiftByMetric.GetOrder(), f: timeShiftByMetric.New}, {name: "timeSlice", filename: "timeSlice", order: timeSlice.GetOrder(), f: timeSlice.New}, {name: "timeStack", filename: "timeStack", order: timeStack.GetOrder(), f: timeStack.New}, + {name: "toLowerCase", filename: "toLowerCase", order: toLowerCase.GetOrder(), f: toLowerCase.New}, + {name: "toUpperCase", filename: "toUpperCase", order: toUpperCase.GetOrder(), f: toUpperCase.New}, {name: "transformNull", filename: "transformNull", order: transformNull.GetOrder(), f: transformNull.New}, {name: "tukey", filename: "tukey", order: tukey.GetOrder(), f: tukey.New}, {name: "unique", filename: "unique", order: unique.GetOrder(), f: unique.New}, diff --git a/expr/functions/highestLowest/function.go b/expr/functions/highestLowest/function.go index 2bc1e4d43..e08ec0ff2 100644 --- a/expr/functions/highestLowest/function.go +++ b/expr/functions/highestLowest/function.go @@ -4,6 +4,7 @@ import ( "container/heap" "context" "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "strings" @@ -88,7 +89,7 @@ func (f *highest) Do(ctx context.Context, e parser.Expr, from, until int64, valu var ok bool compute, ok = consolidations.ConsolidationToFunc[consolidation] if !ok { - return nil, fmt.Errorf("unsupported consolidation function %v", consolidation) + return nil, errors.ErrUnsupportedConsolidationFunction{Func: consolidation} } case "highestMax", "lowestMax": compute = consolidations.MaxValue diff --git a/expr/functions/integralWithReset/function.go b/expr/functions/integralWithReset/function.go index 00e289721..ee72727f7 100644 --- a/expr/functions/integralWithReset/function.go +++ b/expr/functions/integralWithReset/function.go @@ -2,10 +2,10 @@ package integralWithReset import ( "context" + "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "math" - "github.com/ansel1/merry" - "github.com/go-graphite/carbonapi/expr/helper" "github.com/go-graphite/carbonapi/expr/interfaces" "github.com/go-graphite/carbonapi/expr/types" @@ -41,13 +41,13 @@ func (f *integralWithReset) Do(ctx context.Context, e parser.Expr, from, until i return nil, err } if len(resettingSeriesList) != 1 { - return nil, types.ErrWildcardNotAllowed + return nil, errors.ErrWildcardNotAllowed{Target: e.Target(), Arg: e.Arg(1).ToString()} } resettingSeries := resettingSeriesList[0] for _, a := range arg { if a.StepTime != resettingSeries.StepTime || len(a.Values) != len(resettingSeries.Values) { - return nil, merry.Errorf("series %s must have the same length as %s", a.Name, resettingSeries.Name) + return nil, errors.ErrBadData{Target: e.Target(), Msg: fmt.Sprintf("series %s must have the same length as %s", a.Name, resettingSeries.Name)} } } diff --git a/expr/functions/join/function.go b/expr/functions/join/function.go index fef291e17..ce47588ae 100644 --- a/expr/functions/join/function.go +++ b/expr/functions/join/function.go @@ -2,7 +2,7 @@ package join import ( "context" - "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "strings" "github.com/go-graphite/carbonapi/expr/helper" @@ -102,7 +102,7 @@ func (_ *join) Do(ctx context.Context, e parser.Expr, from, until int64, values case sub: return doSub(seriesA, seriesB), nil default: - return nil, fmt.Errorf("unknown join type: %s", joinType) + return nil, errors.ErrInvalidArgument{Msg: joinType} } } diff --git a/expr/functions/kolmogorovSmirnovTest2/function.go b/expr/functions/kolmogorovSmirnovTest2/function.go index f9161c089..33c0bca72 100644 --- a/expr/functions/kolmogorovSmirnovTest2/function.go +++ b/expr/functions/kolmogorovSmirnovTest2/function.go @@ -2,6 +2,7 @@ package kolmogorovSmirnovTest2 import ( "context" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "github.com/dgryski/go-onlinestats" @@ -43,7 +44,7 @@ func (f *kolmogorovSmirnovTest2) Do(ctx context.Context, e parser.Expr, from, un } if len(arg1) != 1 || len(arg2) != 1 { - return nil, types.ErrWildcardNotAllowed + return nil, errors.ErrWildcardNotAllowed{Target: e.Target(), Arg: e.Arg(1).ToString()} } a1 := arg1[0] diff --git a/expr/functions/moving/function.go b/expr/functions/moving/function.go index b5aa6c5bb..53809a029 100644 --- a/expr/functions/moving/function.go +++ b/expr/functions/moving/function.go @@ -2,6 +2,7 @@ package moving import ( "context" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "strconv" @@ -77,7 +78,7 @@ func (f *moving) Do(ctx context.Context, e parser.Expr, from, until int64, value var xFilesFactor float64 if e.ArgsLen() < 2 { - return nil, parser.ErrMissingArgument + return nil, errors.ErrMissingArgument{Target: e.Target()} } switch e.Arg(1).Type() { @@ -93,7 +94,7 @@ func (f *moving) Do(ctx context.Context, e parser.Expr, from, until int64, value n = int(n32) scaleByStep = true default: - err = parser.ErrBadType + err = errors.ErrBadType{Arg: e.Arg(1).ToString(), Exp: parser.TypeToString(parser.EtConst) + " or " + parser.TypeToString(parser.EtString), Got: parser.TypeToString(e.Args()[1].Type())} } if err != nil { return nil, err diff --git a/expr/functions/movingMedian/function.go b/expr/functions/movingMedian/function.go index 3a2ffc706..18a0c52dc 100644 --- a/expr/functions/movingMedian/function.go +++ b/expr/functions/movingMedian/function.go @@ -2,6 +2,7 @@ package movingMedian import ( "context" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "strconv" @@ -85,8 +86,9 @@ func (f *movingMedian) Do(ctx context.Context, e parser.Expr, from, until int64, argstr = "'" + e.Arg(1).StringValue() + "'" scaleByStep = true default: - err = parser.ErrBadType + err = errors.ErrBadType{Arg: e.Arg(1).ToString(), Exp: parser.TypeToString(parser.EtConst) + " or " + parser.TypeToString(parser.EtString), Got: parser.TypeToString(e.Args()[1].Type())} } + if err != nil { return nil, err } diff --git a/expr/functions/nonNegativeDerivative/function.go b/expr/functions/nonNegativeDerivative/function.go index b5642e95e..368afba15 100644 --- a/expr/functions/nonNegativeDerivative/function.go +++ b/expr/functions/nonNegativeDerivative/function.go @@ -2,7 +2,7 @@ package nonNegativeDerivative import ( "context" - "errors" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "strconv" @@ -48,7 +48,7 @@ func (f *nonNegativeDerivative) Do(ctx context.Context, e parser.Expr, from, unt hasMin := !math.IsNaN(minValue) if hasMax && hasMin && maxValue <= minValue { - return nil, errors.New("minValue must be lower than maxValue") + return nil, errors.ErrBadData{Target: e.Target(), Msg: "minValue (" + e.Arg(2).ToString() + ") must be lower than maxValue (" + e.Arg(1).ToString()} } if hasMax && !hasMin { minValue = 0 diff --git a/expr/functions/pearson/function.go b/expr/functions/pearson/function.go index dad3764d7..65a830d19 100644 --- a/expr/functions/pearson/function.go +++ b/expr/functions/pearson/function.go @@ -2,6 +2,7 @@ package pearson import ( "context" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "github.com/dgryski/go-onlinestats" @@ -42,7 +43,7 @@ func (f *pearson) Do(ctx context.Context, e parser.Expr, from, until int64, valu } if len(arg1) != 1 || len(arg2) != 1 { - return nil, types.ErrWildcardNotAllowed + return nil, errors.ErrWildcardNotAllowed{Target: e.Target(), Arg: e.Arg(1).ToString()} } a1 := arg1[0] diff --git a/expr/functions/pearsonClosest/function.go b/expr/functions/pearsonClosest/function.go index 614eefcaf..b44f51866 100644 --- a/expr/functions/pearsonClosest/function.go +++ b/expr/functions/pearsonClosest/function.go @@ -3,7 +3,7 @@ package pearsonClosest import ( "container/heap" "context" - "errors" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "github.com/dgryski/go-onlinestats" @@ -43,7 +43,7 @@ func (f *pearsonClosest) Do(ctx context.Context, e parser.Expr, from, until int6 } if len(ref) != 1 { // TODO(nnuss) error("First argument must be single reference series") - return nil, types.ErrWildcardNotAllowed + return nil, errors.ErrWildcardNotAllowed{Target: e.Target(), Arg: e.Arg(0).ToString()} } compare, err := helper.GetSeriesArg(ctx, e.Arg(1), from, until, values) @@ -61,7 +61,7 @@ func (f *pearsonClosest) Do(ctx context.Context, e parser.Expr, from, until int6 return nil, err } if direction != "pos" && direction != "neg" && direction != "abs" { - return nil, errors.New("direction must be one of: pos, neg, abs") + return nil, errors.ErrInvalidArgument{Target: e.Target(), Msg: "direction " + direction + " must be one of: pos, neg, abs"} } // NOTE: if direction == "abs" && len(compare) <= n : we'll still do the work to rank them diff --git a/expr/functions/perSecond/function.go b/expr/functions/perSecond/function.go index 434861180..20bbaf12e 100644 --- a/expr/functions/perSecond/function.go +++ b/expr/functions/perSecond/function.go @@ -2,7 +2,7 @@ package perSecond import ( "context" - "errors" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "strconv" @@ -50,7 +50,7 @@ func (f *perSecond) Do(ctx context.Context, e parser.Expr, from, until int64, va hasMin := !math.IsNaN(minValue) if hasMax && hasMin && maxValue <= minValue { - return nil, errors.New("minValue must be lower than maxValue") + return nil, errors.ErrBadData{Target: e.Target(), Msg: "minValue (" + e.Arg(2).ToString() + ") must be lower than maxValue (" + e.Arg(1).ToString()} } if hasMax && !hasMin { minValue = 0 diff --git a/expr/functions/polyfit/function.go b/expr/functions/polyfit/function.go index dbbecaf47..edf5f32e3 100644 --- a/expr/functions/polyfit/function.go +++ b/expr/functions/polyfit/function.go @@ -2,7 +2,7 @@ package polyfit import ( "context" - "errors" + "fmt" "math" "strconv" @@ -10,6 +10,7 @@ import ( "github.com/go-graphite/carbonapi/expr/helper" "github.com/go-graphite/carbonapi/expr/interfaces" "github.com/go-graphite/carbonapi/expr/types" + "github.com/go-graphite/carbonapi/pkg/errors" "github.com/go-graphite/carbonapi/pkg/parser" "gonum.org/v1/gonum/mat" ) @@ -45,7 +46,7 @@ func (f *polyfit) Do(ctx context.Context, e parser.Expr, from, until int64, valu if err != nil { return nil, err } else if degree < 1 { - return nil, errors.New("degree must be larger or equal to 1") + return nil, errors.ErrInvalidArgument{Target: e.Target(), Msg: fmt.Sprintf("degree %d must be larger or equal to 1", degree)} } degreeStr := strconv.Itoa(degree) diff --git a/expr/functions/reduce/function.go b/expr/functions/reduce/function.go index 7f71175a3..2dc8938a4 100644 --- a/expr/functions/reduce/function.go +++ b/expr/functions/reduce/function.go @@ -2,6 +2,7 @@ package reduce import ( "context" + "github.com/go-graphite/carbonapi/pkg/errors" "github.com/go-graphite/carbonapi/expr/helper" "github.com/go-graphite/carbonapi/expr/interfaces" @@ -33,7 +34,7 @@ func (f *reduce) Do(ctx context.Context, e parser.Expr, from, until int64, value const matchersStartIndex = 3 if e.ArgsLen() < matchersStartIndex+1 { - return nil, parser.ErrMissingArgument + return nil, errors.ErrMissingArgument{Target: e.Target()} } seriesList, err := helper.GetSeriesArg(ctx, e.Arg(0), from, until, values) diff --git a/expr/functions/seriesList/function.go b/expr/functions/seriesList/function.go index 59ba92029..0e148c66a 100644 --- a/expr/functions/seriesList/function.go +++ b/expr/functions/seriesList/function.go @@ -23,7 +23,7 @@ func GetOrder() interfaces.Order { func New(configFile string) []interfaces.FunctionMetadata { res := make([]interfaces.FunctionMetadata, 0) f := &seriesList{} - functions := []string{"divideSeriesLists", "diffSeriesLists", "multiplySeriesLists", "powSeriesLists"} + functions := []string{"divideSeriesLists", "diffSeriesLists", "multiplySeriesLists", "powSeriesLists", "sumSeriesLists"} for _, n := range functions { res = append(res, interfaces.FunctionMetadata{Name: n, F: f}) } @@ -31,6 +31,10 @@ func New(configFile string) []interfaces.FunctionMetadata { } func (f *seriesList) Do(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) { + if e.ArgsLen() < 2 { + return nil, parser.ErrMissingArgument + } + useConstant := false useDenom := false @@ -85,6 +89,8 @@ func (f *seriesList) Do(ctx context.Context, e parser.Expr, from, until int64, v compute = func(l, r float64) float64 { return l - r } case "powSeriesLists": compute = math.Pow + case "sumSeriesLists": + compute = func(l, r float64) float64 { return l + r } } if useConstant { @@ -306,5 +312,33 @@ func (f *seriesList) Description() map[string]types.FunctionDescription { TagsChange: true, // name tag changed ValuesChange: true, // values changed }, + "sumSeriesLists": { + Description: "Iterates over a two lists and subtracts series lists 2 through n from series 1 list1[0] to list2[0], list1[1] to list2[1] and so on. \n The lists will need to be the same length\nCarbonAPI-specific extension allows to specify default value as 3rd optional argument in case series doesn't exist or value is missing Example:\n\n.. code-block:: none\n\n &target=sumSeriesLists(mining.{carbon,graphite,diamond}.extracted,mining.{carbon,graphite,diamond}.shipped)\n\n", + Function: "sumSeriesLists(seriesListFirstPos, seriesListSecondPos)", + Group: "Combine", + Module: "graphite.render.functions.custom", + Name: "sumSeriesLists", + Params: []types.FunctionParam{ + { + Name: "seriesListFirstPos", + Required: true, + Type: types.SeriesList, + }, + { + Name: "seriesListSecondPos", + Required: true, + Type: types.SeriesList, + }, + { + Name: "default", + Required: false, + Type: types.Float, + }, + }, + SeriesChange: true, // function aggregate metrics or change series items count + NameChange: true, // name changed + TagsChange: true, // name tag changed + ValuesChange: true, // values changed + }, } } diff --git a/expr/functions/seriesList/function_test.go b/expr/functions/seriesList/function_test.go index 9c7f1e720..0e8b4082a 100644 --- a/expr/functions/seriesList/function_test.go +++ b/expr/functions/seriesList/function_test.go @@ -35,6 +35,15 @@ func TestFunction(t *testing.T) { []*types.MetricData{types.MakeMetricData("diffSeries(metric1,metric2)", []float64{-1, math.NaN(), math.NaN(), math.NaN(), 4, 6}, 1, now32)}, }, + { + "sumSeriesLists(metric1,metric2)", + map[parser.MetricRequest][]*types.MetricData{ + {"metric1", 0, 1}: {types.MakeMetricData("metric1", []float64{1, math.NaN(), math.NaN(), 3, 4, 12}, 1, now32)}, + {"metric2", 0, 1}: {types.MakeMetricData("metric2", []float64{2, math.NaN(), 3, math.NaN(), 0, 6}, 1, now32)}, + }, + []*types.MetricData{types.MakeMetricData("sumSeries(metric1,metric2)", + []float64{3, math.NaN(), math.NaN(), math.NaN(), 4, 18}, 1, now32)}, + }, } for _, tt := range tests { @@ -109,6 +118,20 @@ func TestSeriesListMultiReturn(t *testing.T) { "diffSeries(metric1,metric1)": {types.MakeMetricData("diffSeries(metric1,metric1)", []float64{0, 0, 0, 0, 0}, 1, now32)}, }, }, + { + "sumSeriesLists(metric[12],metric[12])", + map[parser.MetricRequest][]*types.MetricData{ + {"metric[12]", 0, 1}: { + types.MakeMetricData("metric1", []float64{1, 2, 3, 4, 5}, 1, now32), + types.MakeMetricData("metric2", []float64{2, 4, 6, 8, 10}, 1, now32), + }, + }, + "sumSeriesListSameGroups", + map[string][]*types.MetricData{ + "sumSeries(metric1,metric1)": {types.MakeMetricData("sumSeries(metric1,metric1)", []float64{2, 4, 6, 8, 10}, 1, now32)}, + "sumSeries(metric2,metric2)": {types.MakeMetricData("sumSeries(metric2,metric2)", []float64{4, 8, 12, 16, 20}, 1, now32)}, + }, + }, } for _, tt := range tests { diff --git a/expr/functions/sortBy/function_test.go b/expr/functions/sortBy/function_test.go index d846b9d4e..dd8298857 100644 --- a/expr/functions/sortBy/function_test.go +++ b/expr/functions/sortBy/function_test.go @@ -1,7 +1,7 @@ package sortBy import ( - "github.com/go-graphite/carbonapi/expr/consolidations" + "github.com/go-graphite/carbonapi/pkg/errors" "testing" "time" @@ -142,7 +142,7 @@ func TestErrorInvalidConsolidationFunction(t *testing.T) { }, }, nil, - consolidations.ErrInvalidConsolidationFunc, + errors.ErrUnsupportedConsolidationFunction{Func: "test"}, }, } diff --git a/expr/functions/substr/function.go b/expr/functions/substr/function.go index b87f7bcf8..a7799ba43 100644 --- a/expr/functions/substr/function.go +++ b/expr/functions/substr/function.go @@ -2,12 +2,12 @@ package substr import ( "context" - "errors" "strings" "github.com/go-graphite/carbonapi/expr/helper" "github.com/go-graphite/carbonapi/expr/interfaces" "github.com/go-graphite/carbonapi/expr/types" + "github.com/go-graphite/carbonapi/pkg/errors" "github.com/go-graphite/carbonapi/pkg/parser" ) @@ -61,7 +61,7 @@ func (f *substr) Do(ctx context.Context, e parser.Expr, from, until int64, value } } if realStartField > len(nodes)-1 { - return nil, errors.New("start out of range") + return nil, errors.ErrBadData{Target: e.Target(), Msg: "start out of range"} } nodes = nodes[realStartField:] } @@ -73,7 +73,7 @@ func (f *substr) Do(ctx context.Context, e parser.Expr, from, until int64, value realStopField = realStopField - realStartField } if realStopField < 0 { - return nil, errors.New("stop out of range") + return nil, errors.ErrBadData{Target: e.Target(), Msg: "stop out of range"} } nodes = nodes[:realStopField] } diff --git a/expr/functions/timeFunction/function.go b/expr/functions/timeFunction/function.go index 5a5c1c596..e6c133c6c 100644 --- a/expr/functions/timeFunction/function.go +++ b/expr/functions/timeFunction/function.go @@ -2,7 +2,7 @@ package timeFunction import ( "context" - "errors" + "github.com/go-graphite/carbonapi/pkg/errors" "github.com/go-graphite/carbonapi/expr/interfaces" "github.com/go-graphite/carbonapi/expr/types" @@ -39,7 +39,7 @@ func (f *timeFunction) Do(ctx context.Context, e parser.Expr, from, until int64, return nil, err } if stepInt <= 0 { - return nil, errors.New("step can't be less than 0") + return nil, errors.ErrBadData{Target: e.Target(), Msg: "step " + e.Arg(1).ToString() + " can't be less than 0"} } step := int64(stepInt) diff --git a/expr/functions/timeShiftByMetric/function.go b/expr/functions/timeShiftByMetric/function.go index 379915c95..b3f49640f 100644 --- a/expr/functions/timeShiftByMetric/function.go +++ b/expr/functions/timeShiftByMetric/function.go @@ -2,12 +2,12 @@ package timeShiftByMetric import ( "context" + "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "regexp" "strings" - "github.com/ansel1/merry" - "github.com/go-graphite/carbonapi/expr/helper" "github.com/go-graphite/carbonapi/expr/interfaces" "github.com/go-graphite/carbonapi/expr/types" @@ -38,7 +38,7 @@ func (f *timeShiftByMetric) Do(ctx context.Context, e parser.Expr, from, until i return nil, err } - latestMarks, err := f.locateLatestMarks(params) + latestMarks, err := f.locateLatestMarks(e.Target(), params) if err != nil { return nil, err } @@ -132,17 +132,17 @@ func (f *timeShiftByMetric) extractCallParams(ctx context.Context, e parser.Expr } for name, dataSet := range dataSets { if len(dataSet) < 2 { - return nil, merry.WithMessagef(errTooFewDatasets, "bad data: need at least 2 %s data sets to process, got %d", name, len(dataSet)) + return nil, errors.ErrBadData{Target: e.Target(), Msg: fmt.Sprintf("need at least 2 %s data sets to process, got %d", name, len(dataSet))} } for _, series := range dataSet { if pointsQty == -1 { pointsQty = len(series.Values) if pointsQty == 0 { - return nil, merry.WithMessagef(errEmptySeries, "bad data: empty series %s", series.Name) + return nil, errors.ErrBadData{Target: e.Target(), Msg: fmt.Sprintf("empty series %s", series.Name)} } } else if pointsQty != len(series.Values) { - return nil, merry.WithMessagef(errSeriesLengthMismatch, "bad data: length of Values for series %s differs from others", series.Name) + return nil, errors.ErrBadData{Target: e.Target(), Msg: fmt.Sprintf("length of Values for series %s differs from others", series.Name)} } if stepTime == -1 { @@ -167,7 +167,7 @@ var reLocateMark *regexp.Regexp = regexp.MustCompile(`(\d+)_(\d+)`) // and looks for the latest ones by _major_ versions // e.g. among set [63_0, 64_0, 64_1, 64_2, 65_0, 65_1] it locates 63_0, 64_4 and 65_1 // returns located elements -func (f *timeShiftByMetric) locateLatestMarks(params *callParams) (versionInfos, error) { +func (f *timeShiftByMetric) locateLatestMarks(target string, params *callParams) (versionInfos, error) { versions := make(versionInfos, 0, len(params.marks)) @@ -206,7 +206,7 @@ func (f *timeShiftByMetric) locateLatestMarks(params *callParams) (versionInfos, // obtain top versions for each major version result := versions.HighestVersions() if len(result) < 2 { - return nil, merry.WithMessagef(errLessThan2Marks, "bad data: could not find 2 marks, only %d found", len(result)) + return nil, errors.ErrBadData{Target: target, Msg: fmt.Sprintf("could not find 2 marks, only %d found", len(result))} } else { return result, nil } diff --git a/expr/functions/timeShiftByMetric/function_test.go b/expr/functions/timeShiftByMetric/function_test.go index 6f876e0c6..5c43fbc2e 100644 --- a/expr/functions/timeShiftByMetric/function_test.go +++ b/expr/functions/timeShiftByMetric/function_test.go @@ -9,6 +9,7 @@ import ( "github.com/go-graphite/carbonapi/expr/helper" "github.com/go-graphite/carbonapi/expr/metadata" "github.com/go-graphite/carbonapi/expr/types" + "github.com/go-graphite/carbonapi/pkg/errors" "github.com/go-graphite/carbonapi/pkg/parser" th "github.com/go-graphite/carbonapi/tests" ) @@ -113,7 +114,7 @@ func TestBadMarks(t *testing.T) { }, }, nil, - errLessThan2Marks, + errors.ErrBadData{}, }, } @@ -149,7 +150,7 @@ func TestNotEnoughSeries(t *testing.T) { parser.MetricRequest{"apps.mark.*", 0, 1}: marksData, }, nil, - errTooFewDatasets, + errors.ErrBadData{}, }) } @@ -172,7 +173,7 @@ func TestNotEnoughSeries(t *testing.T) { parser.MetricRequest{"apps.mark.*", 0, 1}: marksData, }, nil, - errTooFewDatasets, + errors.ErrBadData{}, }) } diff --git a/expr/functions/timeShiftByMetric/misc.go b/expr/functions/timeShiftByMetric/misc.go index 0f467d21a..333c7d492 100644 --- a/expr/functions/timeShiftByMetric/misc.go +++ b/expr/functions/timeShiftByMetric/misc.go @@ -4,18 +4,9 @@ import ( "sort" "strconv" - "github.com/ansel1/merry" - "github.com/go-graphite/carbonapi/expr/types" ) -var ( - errSeriesLengthMismatch = merry.Errorf("bad data: length of Values for series differs from others") - errTooFewDatasets = merry.Errorf("bad data: too few data sets") - errLessThan2Marks = merry.Errorf("bad data: could not find 2 marks") - errEmptySeries = merry.Errorf("bad data: empty series") -) - type callParams struct { marks []*types.MetricData metrics []*types.MetricData diff --git a/expr/functions/toLowerCase/function.go b/expr/functions/toLowerCase/function.go new file mode 100644 index 000000000..db72fc8ab --- /dev/null +++ b/expr/functions/toLowerCase/function.go @@ -0,0 +1,96 @@ +package toLowerCase + +import ( + "context" + "github.com/go-graphite/carbonapi/expr/helper" + "strings" + + "github.com/go-graphite/carbonapi/expr/interfaces" + "github.com/go-graphite/carbonapi/expr/types" + "github.com/go-graphite/carbonapi/pkg/parser" +) + +type toLowerCase struct { + interfaces.FunctionBase +} + +func GetOrder() interfaces.Order { + return interfaces.Any +} + +func New(configFile string) []interfaces.FunctionMetadata { + res := make([]interfaces.FunctionMetadata, 0) + f := &toLowerCase{} + functions := []string{"toLowerCase"} + for _, n := range functions { + res = append(res, interfaces.FunctionMetadata{Name: n, F: f}) + } + return res +} + +// toLowerCase(seriesList, *pos) +func (f *toLowerCase) Do(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) { + args, err := helper.GetSeriesArg(ctx, e.Args()[0], from, until, values) + if err != nil { + return nil, err + } + + var pos []int + + if e.ArgsLen() >= 2 { + pos, err = e.GetIntArgs(1) + if err != nil { + return nil, err + } + } + + results := make([]*types.MetricData, 0, len(args)+1) + + for _, a := range args { + r := a.CopyLink() + + if len(pos) == 0 { + r.Name = strings.ToLower(a.Name) + } else { + for _, i := range pos { + if i < 0 { // Handle negative indices by indexing backwards + i = len(r.Name) + i + } + lowered := strings.ToLower(string(r.Name[i])) + r.Name = r.Name[:i] + lowered + r.Name[i+1:] + } + } + + results = append(results, r) + } + + return results, nil +} + +// Description is auto-generated description, based on output of https://github.com/graphite-project/graphite-web +func (f *toLowerCase) Description() map[string]types.FunctionDescription { + return map[string]types.FunctionDescription{ + "toLowerCase": { + Description: "Takes one metric or a wildcard seriesList and lowers the case of each letter. \n Optionally, a letter position to lower case can be specified, in which case only the letter at the specified position gets lower-cased.\n The position parameter may be given multiple times. The position parameter may be negative to define a position relative to the end of the metric name.", + Function: "toLowerCase(seriesList, *pos)", + Group: "Alias", + Module: "graphite.render.functions", + Name: "toLowerCase", + Params: []types.FunctionParam{ + { + Name: "seriesList", + Required: true, + Type: types.SeriesList, + }, + { + Multiple: true, + Name: "pos", + Type: types.Node, + Required: false, + }, + }, + NameChange: true, // name changed + ValuesChange: true, // values changed + }, + } +} diff --git a/expr/functions/toLowerCase/function_test.go b/expr/functions/toLowerCase/function_test.go new file mode 100644 index 000000000..7f8c1e2f0 --- /dev/null +++ b/expr/functions/toLowerCase/function_test.go @@ -0,0 +1,69 @@ +package toLowerCase + +import ( + "math" + "testing" + "time" + + "github.com/go-graphite/carbonapi/expr/helper" + "github.com/go-graphite/carbonapi/expr/metadata" + "github.com/go-graphite/carbonapi/expr/types" + "github.com/go-graphite/carbonapi/pkg/parser" + th "github.com/go-graphite/carbonapi/tests" +) + +func init() { + md := New("") + evaluator := th.EvaluatorFromFunc(md[0].F) + metadata.SetEvaluator(evaluator) + helper.SetEvaluator(evaluator) + for _, m := range md { + metadata.RegisterFunction(m.Name, m.F) + } +} + +func TestToLowerCaseFunction(t *testing.T) { + now32 := int64(time.Now().Unix()) + + tests := []th.EvalTestItem{ + { + "toLowerCase(METRIC.TEST.FOO)", + map[parser.MetricRequest][]*types.MetricData{ + {"METRIC.TEST.FOO", 0, 1}: {types.MakeMetricData("METRIC.TEST.FOO", []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + []*types.MetricData{types.MakeMetricData("metric.test.foo", + []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + { + "toLowerCase(METRIC.TEST.FOO,7)", + map[parser.MetricRequest][]*types.MetricData{ + {"METRIC.TEST.FOO", 0, 1}: {types.MakeMetricData("METRIC.TEST.FOO", []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + []*types.MetricData{types.MakeMetricData("METRIC.tEST.FOO", + []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + { + "toLowerCase(METRIC.TEST.FOO,-3)", + map[parser.MetricRequest][]*types.MetricData{ + {"METRIC.TEST.FOO", 0, 1}: {types.MakeMetricData("METRIC.TEST.FOO", []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + []*types.MetricData{types.MakeMetricData("METRIC.TEST.fOO", + []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + { + "toLowerCase(METRIC.TEST.FOO,0,7,12)", + map[parser.MetricRequest][]*types.MetricData{ + {"METRIC.TEST.FOO", 0, 1}: {types.MakeMetricData("METRIC.TEST.FOO", []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + []*types.MetricData{types.MakeMetricData("mETRIC.tEST.fOO", + []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + } + + for _, tt := range tests { + testName := tt.Target + t.Run(testName, func(t *testing.T) { + th.TestEvalExpr(t, &tt) + }) + } +} diff --git a/expr/functions/toUpperCase/function.go b/expr/functions/toUpperCase/function.go new file mode 100644 index 000000000..d3a0330f7 --- /dev/null +++ b/expr/functions/toUpperCase/function.go @@ -0,0 +1,95 @@ +package toUpperCase + +import ( + "context" + "github.com/go-graphite/carbonapi/expr/helper" + "strings" + + "github.com/go-graphite/carbonapi/expr/interfaces" + "github.com/go-graphite/carbonapi/expr/types" + "github.com/go-graphite/carbonapi/pkg/parser" +) + +type toUpperCase struct { + interfaces.FunctionBase +} + +func GetOrder() interfaces.Order { + return interfaces.Any +} + +func New(configFile string) []interfaces.FunctionMetadata { + res := make([]interfaces.FunctionMetadata, 0) + f := &toUpperCase{} + functions := []string{"toLowerCase"} + for _, n := range functions { + res = append(res, interfaces.FunctionMetadata{Name: n, F: f}) + } + return res +} + +// toUpperCase(seriesList, *pos) +func (f *toUpperCase) Do(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) { + args, err := helper.GetSeriesArg(ctx, e.Args()[0], from, until, values) + if err != nil { + return nil, err + } + + var pos []int + + if e.ArgsLen() >= 2 { + pos, err = e.GetIntArgs(1) + if err != nil { + return nil, err + } + } + + results := make([]*types.MetricData, 0, len(args)+1) + + for _, a := range args { + r := a.CopyLink() + + if len(pos) == 0 { + r.Name = strings.ToUpper(a.Name) + } else { + for _, i := range pos { + if i < 0 { // Handle negative indices by indexing backwards + i = len(r.Name) + i + } + uppered := strings.ToUpper(string(r.Name[i])) + r.Name = r.Name[:i] + uppered + r.Name[i+1:] + } + } + + results = append(results, r) + } + + return results, nil +} + +// Description is auto-generated description, based on output of https://github.com/graphite-project/graphite-web +func (f *toUpperCase) Description() map[string]types.FunctionDescription { + return map[string]types.FunctionDescription{ + "toUpperCase": { + Description: "Takes one metric or a wildcard seriesList and uppers the case of each letter. \n Optionally, a letter position to upper case can be specified, in which case only the letter at the specified position gets upper-cased.\n The position parameter may be given multiple times. The position parameter may be negative to define a position relative to the end of the metric name.", + Function: "toUpperCase(seriesList, *pos)", + Group: "Alias", + Module: "graphite.render.functions", + Name: "toUpperCase", + Params: []types.FunctionParam{ + { + Name: "seriesList", + Required: true, + Type: types.SeriesList, + }, + { + Multiple: true, + Name: "pos", + Type: types.Node, + }, + }, + NameChange: true, // name changed + ValuesChange: true, // values changed + }, + } +} diff --git a/expr/functions/toUpperCase/function_test.go b/expr/functions/toUpperCase/function_test.go new file mode 100644 index 000000000..880618d1f --- /dev/null +++ b/expr/functions/toUpperCase/function_test.go @@ -0,0 +1,69 @@ +package toUpperCase + +import ( + "math" + "testing" + "time" + + "github.com/go-graphite/carbonapi/expr/helper" + "github.com/go-graphite/carbonapi/expr/metadata" + "github.com/go-graphite/carbonapi/expr/types" + "github.com/go-graphite/carbonapi/pkg/parser" + th "github.com/go-graphite/carbonapi/tests" +) + +func init() { + md := New("") + evaluator := th.EvaluatorFromFunc(md[0].F) + metadata.SetEvaluator(evaluator) + helper.SetEvaluator(evaluator) + for _, m := range md { + metadata.RegisterFunction(m.Name, m.F) + } +} + +func TestToUpperCaseFunction(t *testing.T) { + now32 := int64(time.Now().Unix()) + + tests := []th.EvalTestItem{ + { + "toUpperCase(metric.test.foo)", + map[parser.MetricRequest][]*types.MetricData{ + {"metric.test.foo", 0, 1}: {types.MakeMetricData("metric.test.foo", []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + []*types.MetricData{types.MakeMetricData("METRIC.TEST.FOO", + []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + { + "toUpperCase(metric.test.foo,7)", + map[parser.MetricRequest][]*types.MetricData{ + {"metric.test.foo", 0, 1}: {types.MakeMetricData("metric.test.foo", []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + []*types.MetricData{types.MakeMetricData("metric.Test.foo", + []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + { + "toLowerCase(metric.test.foo,-3)", + map[parser.MetricRequest][]*types.MetricData{ + {"metric.test.foo", 0, 1}: {types.MakeMetricData("metric.test.foo", []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + []*types.MetricData{types.MakeMetricData("metric.test.Foo", + []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + { + "toUpperCase(metric.test.foo,0,7,12)", + map[parser.MetricRequest][]*types.MetricData{ + {"metric.test.foo", 0, 1}: {types.MakeMetricData("metric.test.foo", []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + []*types.MetricData{types.MakeMetricData("Metric.Test.Foo", + []float64{1, 2, 0, 7, 8, 20, 30, math.NaN()}, 1, now32)}, + }, + } + + for _, tt := range tests { + testName := tt.Target + t.Run(testName, func(t *testing.T) { + th.TestEvalExpr(t, &tt) + }) + } +} diff --git a/expr/functions/transformNull/function.go b/expr/functions/transformNull/function.go index 8c2f23668..68f95cbfa 100644 --- a/expr/functions/transformNull/function.go +++ b/expr/functions/transformNull/function.go @@ -3,6 +3,7 @@ package transformNull import ( "context" "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "strconv" @@ -65,11 +66,11 @@ func (f *transformNull) Do(ctx context.Context, e parser.Expr, from, until int64 } if len(referenceSeries) == 0 { - return nil, fmt.Errorf("reference series is not a valid metric") + return nil, errors.ErrBadData{Target: e.Target(), Msg: fmt.Sprintf("reference series %s is not a valid metric", referenceSeriesExpr.ToString())} } length := len(referenceSeries[0].Values) if length != len(arg[0].Values) { - return nil, fmt.Errorf("length of series and reference series must be the same") + return nil, errors.ErrBadData{Target: e.Target(), Msg: fmt.Sprintf("length of series (len: %d) and reference series (len: %d) must be the same", len(arg[0].Values), length)} } valMap = make([]bool, length) diff --git a/expr/functions/tukey/function.go b/expr/functions/tukey/function.go index 11041476a..38180fda3 100644 --- a/expr/functions/tukey/function.go +++ b/expr/functions/tukey/function.go @@ -3,7 +3,8 @@ package tukey import ( "container/heap" "context" - "errors" + "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "sort" "strings" @@ -49,7 +50,7 @@ func (f *tukey) Do(ctx context.Context, e parser.Expr, from, until int64, values return nil, err } if n < 1 { - return nil, errors.New("n must be larger or equal to 1") + return nil, errors.ErrInvalidArgument{Target: e.Target(), Msg: fmt.Sprintf("n %d must be larger or equal to 1", n)} } var beginInterval int @@ -65,8 +66,9 @@ func (f *tukey) Do(ctx context.Context, e parser.Expr, from, until int64, values beginInterval /= int(arg[0].StepTime) // TODO(nnuss): make sure the arrays are all the same 'size' default: - err = parser.ErrBadType + err = errors.ErrBadType{Arg: e.Arg(1).ToString(), Exp: parser.TypeToString(parser.EtConst) + " or " + parser.TypeToString(parser.EtString), Got: parser.TypeToString(e.Args()[1].Type())} } + if err != nil { return nil, err } diff --git a/expr/functions/verticalLine/function_cairo.go b/expr/functions/verticalLine/function_cairo.go index 3a39f007f..ad194077f 100644 --- a/expr/functions/verticalLine/function_cairo.go +++ b/expr/functions/verticalLine/function_cairo.go @@ -6,6 +6,7 @@ package verticalLine import ( "context" "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "time" "github.com/go-graphite/carbonapi/expr/interfaces" @@ -15,8 +16,6 @@ import ( pb "github.com/go-graphite/protocol/carbonapi_v3_pb" ) -var TsOutOfRangeError = fmt.Errorf("timestamp out of range") - type verticalLine struct { interfaces.FunctionBase } @@ -46,9 +45,9 @@ func (f *verticalLine) Do(_ context.Context, e parser.Expr, from, until int64, _ ts := until + int64(start) if ts < from { - return nil, fmt.Errorf("ts %s is before start %s: %w", time.Unix(ts, 0), time.Unix(from, 0), TsOutOfRangeError) + return nil, errors.ErrTimestampOutOfRange{Target: e.Target(), Msg: fmt.Sprintf("ts %s is before start %s", time.Unix(ts, 0), time.Unix(from, 0))} } else if ts > until { - return nil, fmt.Errorf("ts %s is after end %s: %w", time.Unix(ts, 0), time.Unix(until, 0), TsOutOfRangeError) + return nil, errors.ErrTimestampOutOfRange{Target: e.Target(), Msg: fmt.Sprintf("ts %s is after end %s: %d", time.Unix(ts, 0), time.Unix(until, 0), ts)} } label, err := e.GetStringArgDefault(1, "") diff --git a/expr/functions/verticalLine/function_test.go b/expr/functions/verticalLine/function_test.go index 0fb31101f..af0164b86 100644 --- a/expr/functions/verticalLine/function_test.go +++ b/expr/functions/verticalLine/function_test.go @@ -4,6 +4,7 @@ package verticalLine import ( + "github.com/go-graphite/carbonapi/pkg/errors" "testing" "time" @@ -88,7 +89,7 @@ func TestFunctionErrors(t *testing.T) { {"foo", from, nowUnix}: {types.MakeMetricData("foo", []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}, 1, nowUnix)}, }, []*types.MetricData{}, - TsOutOfRangeError, + errors.ErrTimestampOutOfRange{}, }, { "verticalLine(\"+5m\")", @@ -96,7 +97,7 @@ func TestFunctionErrors(t *testing.T) { {"foo", from, nowUnix}: {types.MakeMetricData("foo", []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}, 1, nowUnix)}, }, []*types.MetricData{}, - TsOutOfRangeError, + errors.ErrTimestampOutOfRange{}, }, } diff --git a/expr/helper/helper.go b/expr/helper/helper.go index 42f2e8ac4..497425af1 100644 --- a/expr/helper/helper.go +++ b/expr/helper/helper.go @@ -2,7 +2,7 @@ package helper import ( "context" - "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "math" "regexp" "strings" @@ -17,13 +17,6 @@ var evaluator interfaces.Evaluator // Backref is a pre-compiled expression for backref var Backref = regexp.MustCompile(`\\(\d+)`) -// ErrUnknownFunction is an error message about unknown function -type ErrUnknownFunction string - -func (e ErrUnknownFunction) Error() string { - return fmt.Sprintf("unknown function in evalExpr: %q", string(e)) -} - // SetEvaluator sets evaluator for all helper functions func SetEvaluator(e interfaces.Evaluator) { evaluator = e @@ -32,7 +25,7 @@ func SetEvaluator(e interfaces.Evaluator) { // GetSeriesArg returns argument from series. func GetSeriesArg(ctx context.Context, arg parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) { if !arg.IsName() && !arg.IsFunc() { - return nil, parser.ErrMissingTimeseries + return nil, errors.ErrMissingTimeseries{Target: arg.Target()} } a, err := evaluator.Eval(ctx, arg, from, until, values) @@ -121,7 +114,7 @@ type seriesFunc1 func(*types.MetricData) *types.MetricData func ForEachSeriesDo1(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData, function seriesFunc1) ([]*types.MetricData, error) { arg, err := GetSeriesArg(ctx, e.Arg(0), from, until, values) if err != nil { - return nil, parser.ErrMissingTimeseries + return nil, errors.ErrMissingTimeseries{Target: e.Target()} } var results []*types.MetricData @@ -137,7 +130,7 @@ type seriesFunc func(*types.MetricData, *types.MetricData) *types.MetricData func ForEachSeriesDo(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData, function seriesFunc) ([]*types.MetricData, error) { arg, err := GetSeriesArg(ctx, e.Arg(0), from, until, values) if err != nil { - return nil, parser.ErrMissingTimeseries + return nil, errors.ErrMissingTimeseries{Target: e.Target()} } var results []*types.MetricData diff --git a/internal/generateFuncs/gen.go b/internal/generateFuncs/gen.go index 96ad1ddb3..0167c6aa0 100644 --- a/internal/generateFuncs/gen.go +++ b/internal/generateFuncs/gen.go @@ -41,6 +41,9 @@ import ( "strings" `) for _, m := range funcs { + if m == "config" { + continue + } fmt.Fprintf(writer, " \"github.com/go-graphite/carbonapi/expr/functions/%s\"\n", m) } fmt.Fprintf(writer, ` "github.com/go-graphite/carbonapi/expr/interfaces" @@ -57,6 +60,9 @@ type initFunc struct { func New(configs map[string]string) { funcs := []initFunc{`) for _, m := range funcs { + if m == "config" { + continue + } fmt.Fprintf(writer, ` {name: "%s", filename: "%s", order: %s.GetOrder(), f: %s.New},`, m, m, m, m) diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go new file mode 100644 index 000000000..f4973894d --- /dev/null +++ b/pkg/errors/errors.go @@ -0,0 +1,206 @@ +package errors + +import ( + "fmt" + "net/http" +) + +// ErrMissingExpr is a parse error returned when an expression is missing. +type ErrMissingExpr string + +func (e ErrMissingExpr) Error() string { + return fmt.Sprintf(string(e)) +} + +func (e ErrMissingExpr) HTTPStatusCode() int { + return http.StatusBadRequest +} + +// ErrMissingComma is a parse error returned when an expression is missing a comma. +type ErrMissingComma string + +func (e ErrMissingComma) Error() string { + return fmt.Sprintf("missing comma: %s", string(e)) +} + +func (e ErrMissingComma) HTTPStatusCode() int { + return http.StatusBadRequest +} + +// ErrMissingQuote is a parse error returned when an expression is missing a quote. +type ErrMissingQuote string + +func (e ErrMissingQuote) Error() string { + return fmt.Sprintf("missing quote: %s", string(e)) +} + +func (e ErrMissingQuote) HTTPStatusCode() int { + return http.StatusBadRequest +} + +// ErrUnexpectedCharacter is a parse error returned when an expression contains an unexpected character. +type ErrUnexpectedCharacter struct { + Expr string + CharNum int + Char string +} + +func (e ErrUnexpectedCharacter) Error() string { + return fmt.Sprintf("unexpected character. string_to_parse=%s character_number=%d character=%s", e.Expr, e.CharNum, e.Char) +} + +func (e ErrUnexpectedCharacter) HTTPStatusCode() int { + return http.StatusBadRequest +} + +// ErrUnknownFunction is an error that is returned when an unknown function is specified in the query +type ErrUnknownFunction string + +func (e ErrUnknownFunction) Error() string { + return fmt.Sprintf("unknown function %q", string(e)) +} + +func (e ErrUnknownFunction) HTTPStatusCode() int { + return http.StatusBadRequest +} + +// ErrBadType is an eval error returned when an argument has the wrong type. +type ErrBadType struct { + Arg string + Exp string + Got string +} + +func (e ErrBadType) Error() string { + return fmt.Sprintf("%q: bad type. expected %q - got %q", e.Arg, e.Exp, e.Got) +} + +func (e ErrBadType) HTTPStatusCode() int { + return http.StatusBadRequest +} + +// ErrMissingArgument is an eval error returned when an argument is missing. +type ErrMissingArgument struct { + Target string +} + +func (e ErrMissingArgument) Error() string { + return fmt.Sprintf("%q: missing argument", e.Target) +} + +func (e ErrMissingArgument) HTTPStatusCode() int { + return http.StatusBadRequest +} + +// ErrMissingTimeseries is an eval error returned when a time series argument is missing. +type ErrMissingTimeseries struct { + Target string +} + +func (e ErrMissingTimeseries) Error() string { + return fmt.Sprintf("%q: missing time series argument", e.Target) +} + +func (e ErrMissingTimeseries) HTTPStatusCode() int { + return http.StatusBadRequest +} + +// ErrUnknownTimeUnits is an eval error returned when a time unit is unknown to system +type ErrUnknownTimeUnits struct { + Target string + Units string +} + +func (e ErrUnknownTimeUnits) Error() string { + return fmt.Sprintf("%s: unknown time units: %s", e.Target, e.Units) +} + +func (e ErrUnknownTimeUnits) HTTPStatusCode() int { + return http.StatusBadRequest +} + +// ErrTimestampOutOfRange +type ErrTimestampOutOfRange struct { + Target string + Msg string +} + +func (e ErrTimestampOutOfRange) Error() string { + return fmt.Sprintf("%s: timestamp out of range: %s", e.Target, e.Msg) +} + +func (e ErrTimestampOutOfRange) HTTPStatusCode() int { + return http.StatusBadRequest +} + +// ErrUnsupportedConsolidationFunction is an eval error returned when a consolidation function is unknown to system +type ErrUnsupportedConsolidationFunction struct { + Func string +} + +func (e ErrUnsupportedConsolidationFunction) Error() string { + return fmt.Sprintf("unknown consolidation function %q", e.Func) +} + +func (e ErrUnsupportedConsolidationFunction) HTTPStatusCode() int { + return http.StatusBadRequest +} + +// ErrBadData +type ErrBadData struct { + Target string + Msg string +} + +func (e ErrBadData) Error() string { + return fmt.Sprintf("%s: bad data: %s", e.Target, e.Msg) +} + +func (e ErrBadData) Message() string { + return e.Msg +} + +func (e ErrBadData) HTTPStatusCode() int { + return http.StatusBadRequest +} + +// ErrWildcardNotAllowed is an eval error returned when a wildcard/glob argument is found where a single series is required. +type ErrWildcardNotAllowed struct { + Target string + Arg string +} + +func (e ErrWildcardNotAllowed) Error() string { + return fmt.Sprintf("%q: found wildcard where series expected %q", e.Target, e.Arg) +} + +func (e ErrWildcardNotAllowed) HTTPStatusCode() int { + return http.StatusBadRequest +} + +// ErrTooManyArguments is an eval error returned when too many arguments are provided. +type ErrTooManyArguments struct { + Target string +} + +func (e ErrTooManyArguments) Error() string { + return fmt.Sprintf("%q: too many arguments", e.Target) +} + +func (e ErrTooManyArguments) HTTPStatusCode() int { + return http.StatusBadRequest +} + +// ErrInvalidArgument +type ErrInvalidArgument struct { + Target string + Msg string +} + +func (e ErrInvalidArgument) Error() string { + return fmt.Sprintf("%s: invalid argument: \" %q", e.Target, e.Msg) +} + +func (e ErrInvalidArgument) HTTPStatusCode() int { + return http.StatusBadRequest +} diff --git a/pkg/parser/interface.go b/pkg/parser/interface.go index 0bc1a4996..13f494880 100644 --- a/pkg/parser/interface.go +++ b/pkg/parser/interface.go @@ -268,3 +268,20 @@ func NewExprTyped(target string, args []Expr) Expr { return e } + +func TypeToString(exprType ExprType) string { + switch exprType { + case EtConst: + return "const" + case EtBool: + return "bool" + case EtString: + return "string" + case EtName: + return "name" + case EtFunc: + return "function" + default: + return "unknown" + } +} diff --git a/pkg/parser/internal.go b/pkg/parser/internal.go index 0be89c0c6..003438bdb 100644 --- a/pkg/parser/internal.go +++ b/pkg/parser/internal.go @@ -2,6 +2,7 @@ package parser import ( "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "strconv" "runtime/debug" @@ -13,7 +14,8 @@ func (e *expr) doGetIntArg() (int, error) { f, err := strconv.ParseInt(e.valStr, 0, 32) return int(f), err } - return 0, ErrBadType + + return 0, errors.ErrBadType{Arg: e.ToString(), Exp: TypeToString(EtConst), Got: TypeToString(e.Type())} } return int(e.val), nil @@ -33,7 +35,7 @@ func (e *expr) doGetFloatArg() (float64, error) { f, err := strconv.ParseFloat(e.valStr, 64) return f, err } - return 0, ErrBadType + return 0, errors.ErrBadType{Arg: e.ToString(), Exp: TypeToString(EtConst), Got: TypeToString(e.Type())} } return e.val, nil @@ -41,7 +43,7 @@ func (e *expr) doGetFloatArg() (float64, error) { func (e *expr) doGetStringArg() (string, error) { if e.etype != EtString { - return "", ErrBadType + return "", errors.ErrBadType{Arg: e.ToString(), Exp: TypeToString(EtString), Got: TypeToString(e.Type())} } return e.valStr, nil @@ -49,7 +51,7 @@ func (e *expr) doGetStringArg() (string, error) { func (e *expr) doGetBoolArg() (bool, error) { if e.etype != EtString && e.etype != EtBool && e.etype != EtConst { - return false, ErrBadType + return false, errors.ErrBadType{Arg: e.ToString(), Exp: TypeToString(EtConst) + " or " + TypeToString(EtBool) + " or " + TypeToString(EtConst), Got: TypeToString(e.Type())} } switch e.valStr { @@ -59,7 +61,7 @@ func (e *expr) doGetBoolArg() (bool, error) { return true, nil } - return false, ErrBadType + return false, errors.ErrBadType{Arg: e.ToString(), Exp: TypeToString(EtConst) + " or " + TypeToString(EtBool) + " or " + TypeToString(EtConst), Got: TypeToString(e.Type())} } func (e *expr) toExpr() interface{} { diff --git a/pkg/parser/interval.go b/pkg/parser/interval.go index 1142035a7..ee227e7c0 100644 --- a/pkg/parser/interval.go +++ b/pkg/parser/interval.go @@ -1,6 +1,7 @@ package parser import ( + "github.com/go-graphite/carbonapi/pkg/errors" "strconv" ) @@ -51,7 +52,7 @@ func IntervalString(s string, defaultSign int) (int32, error) { case "y", "year", "years": units = 365 * 24 * 60 * 60 default: - return 0, ErrUnknownTimeUnits + return 0, errors.ErrUnknownTimeUnits{Units: unitStr} } offset, err := strconv.Atoi(offsetStr) diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index 0b94cfb0e..5358c300f 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -3,6 +3,7 @@ package parser import ( "bytes" "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "strconv" "strings" "unicode" @@ -221,16 +222,16 @@ func (e *expr) Metrics() []MetricRequest { func (e *expr) GetIntervalArg(n, defaultSign int) (int32, error) { if len(e.args) <= n { - return 0, ErrMissingArgument + return 0, errors.ErrMissingArgument{Target: e.Target()} } if e.args[n].etype != EtString { - return 0, ErrBadType + return 0, errors.ErrBadType{Arg: e.args[n].valStr, Exp: TypeToString(EtString), Got: TypeToString(e.args[n].etype)} } seconds, err := IntervalString(e.args[n].valStr, defaultSign) if err != nil { - return 0, ErrBadType + return 0, errors.ErrBadType{Arg: e.args[n].valStr, Exp: TypeToString(EtString), Got: TypeToString(e.args[n].etype)} } return seconds, nil @@ -242,7 +243,7 @@ func (e *expr) GetIntervalNamedOrPosArgDefault(k string, n, defaultSign int, v i if a := e.getNamedArg(k); a != nil { val, err = a.doGetStringArg() if err != nil { - return 0, ErrBadType + return 0, errors.ErrBadType{Arg: a.ToString(), Exp: TypeToString(EtString), Got: TypeToString(e.etype)} } } else { if len(e.args) <= n { @@ -250,14 +251,14 @@ func (e *expr) GetIntervalNamedOrPosArgDefault(k string, n, defaultSign int, v i } if e.args[n].etype != EtString { - return 0, ErrBadType + return 0, errors.ErrBadType{Arg: e.args[n].ToString(), Exp: TypeToString(EtString), Got: TypeToString(e.etype)} } val = e.args[n].valStr } seconds, err := IntervalString(val, defaultSign) if err != nil { - return 0, ErrBadType + return 0, errors.ErrBadType{Arg: val, Exp: TypeToString(EtString), Got: TypeToString(e.etype)} } return int64(seconds), nil @@ -265,7 +266,7 @@ func (e *expr) GetIntervalNamedOrPosArgDefault(k string, n, defaultSign int, v i func (e *expr) GetStringArg(n int) (string, error) { if len(e.args) <= n { - return "", ErrMissingArgument + return "", errors.ErrMissingArgument{Target: e.Target()} } return e.args[n].doGetStringArg() @@ -273,7 +274,7 @@ func (e *expr) GetStringArg(n int) (string, error) { func (e *expr) GetStringArgs(n int) ([]string, error) { if len(e.args) <= n { - return nil, ErrMissingArgument + return nil, errors.ErrMissingArgument{Target: e.Target()} } strs := make([]string, 0, len(e.args)-n) @@ -307,7 +308,7 @@ func (e *expr) GetStringNamedOrPosArgDefault(k string, n int, s string) (string, func (e *expr) GetFloatArg(n int) (float64, error) { if len(e.args) <= n { - return 0, ErrMissingArgument + return 0, errors.ErrMissingArgument{Target: e.Target()} } return e.args[n].doGetFloatArg() @@ -331,7 +332,7 @@ func (e *expr) GetFloatNamedOrPosArgDefault(k string, n int, v float64) (float64 func (e *expr) GetIntArg(n int) (int, error) { if len(e.args) <= n { - return 0, ErrMissingArgument + return 0, errors.ErrMissingArgument{Target: e.Target()} } return e.args[n].doGetIntArg() @@ -339,7 +340,7 @@ func (e *expr) GetIntArg(n int) (int, error) { func (e *expr) GetIntArgs(n int) ([]int, error) { if len(e.args) <= n { - return nil, ErrMissingArgument + return nil, errors.ErrMissingArgument{Target: e.Target()} } ints := make([]int, 0, len(e.args)-n) @@ -411,7 +412,7 @@ func (e *expr) GetBoolArgDefault(n int, b bool) (bool, error) { func (e *expr) GetNodeOrTagArgs(n int, single bool) ([]NodeOrTag, error) { if len(e.args) <= n { - return nil, ErrMissingArgument + return nil, errors.ErrMissingArgument{Target: e.Target()} } nodeTags := make([]NodeOrTag, 0, len(e.args)-n) @@ -466,7 +467,7 @@ func parseExprWithoutPipe(e string) (Expr, string, error) { } if e == "" { - return nil, "", ErrMissingExpr + return nil, "", errors.ErrMissingExpr(string(e)) } if '0' <= e[0] && e[0] <= '9' || e[0] == '-' || e[0] == '+' { @@ -611,7 +612,7 @@ func parseArgList(e string) (string, []*expr, map[string]*expr, string, error) { } if e == "" { - return "", nil, nil, "", ErrMissingComma + return "", nil, nil, "", errors.ErrMissingComma(argString) } // we now know we're parsing a key-value pair @@ -623,11 +624,11 @@ func parseArgList(e string) (string, []*expr, map[string]*expr, string, error) { } if eCont == "" { - return "", nil, nil, "", ErrMissingComma + return "", nil, nil, "", errors.ErrMissingComma(e) } if !argCont.IsConst() && !argCont.IsName() && !argCont.IsString() && !argCont.IsBool() { - return "", nil, nil, eCont, ErrBadType + return "", nil, nil, eCont, errors.ErrBadType{Arg: argCont.ToString(), Exp: TypeToString(EtString), Got: TypeToString(argCont.Type())} } if namedArgs == nil { @@ -675,7 +676,7 @@ func parseArgList(e string) (string, []*expr, map[string]*expr, string, error) { } if e[0] != ',' && e[0] != ' ' { - return "", nil, nil, "", merry.Wrap(ErrUnexpectedCharacter).WithUserMessagef("string_to_parse=`%v`, character_number=%v, character=`%v`", eOrig, charNum, string(e[0])) + return "", nil, nil, "", merry.Wrap(errors.ErrUnexpectedCharacter{Expr: eOrig, CharNum: charNum, Char: string(e[0])}) } e = e[1:] @@ -823,7 +824,7 @@ func parseString(s string) (string, string, error) { } if i == len(s) { - return "", "", ErrMissingQuote + return "", "", errors.ErrMissingQuote(s) } diff --git a/scripts/build-drone-utilities.sh b/scripts/build-drone-utilities.sh new file mode 100755 index 000000000..6556e7866 --- /dev/null +++ b/scripts/build-drone-utilities.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eufo pipefail + +pushd .drone + +go build -o coverage ./cmd/coverage +go build -o ghcomment ./cmd/ghcomment + +popd \ No newline at end of file diff --git a/scripts/generate-drone-yml.sh b/scripts/generate-drone-yml.sh new file mode 100755 index 000000000..d7ce1857f --- /dev/null +++ b/scripts/generate-drone-yml.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -eufo pipefail + +drone jsonnet --source .drone/drone.jsonnet --target .drone/drone.yml --stream --format +drone lint --trusted .drone/drone.yml +drone sign --save grafana/carbonapi .drone/drone.yml \ No newline at end of file diff --git a/tests/helper.go b/tests/helper.go index 92c8e0fd7..4376cf61a 100644 --- a/tests/helper.go +++ b/tests/helper.go @@ -2,13 +2,12 @@ package tests import ( "context" - "fmt" + "github.com/go-graphite/carbonapi/pkg/errors" "reflect" "testing" "time" "github.com/ansel1/merry" - "github.com/go-graphite/carbonapi/expr/helper" "github.com/go-graphite/carbonapi/expr/interfaces" "github.com/go-graphite/carbonapi/expr/metadata" "github.com/go-graphite/carbonapi/expr/types" @@ -38,14 +37,14 @@ func (evaluator *FuncEvaluator) Eval(ctx context.Context, e parser.Expr, from, u // all functions have arguments -- check we do too if e.ArgsLen() == 0 { - return nil, parser.ErrMissingArgument + return nil, errors.ErrMissingArgument{Target: e.Target()} } if evaluator.eval != nil { return evaluator.eval(context.Background(), e, from, until, values) } - return nil, helper.ErrUnknownFunction(e.Target()) + return nil, errors.ErrUnknownFunction(e.Target()) } func DummyEvaluator() interfaces.Evaluator { @@ -70,7 +69,7 @@ func EvaluatorFromFuncWithMetadata(metadata map[string]interfaces.Function) inte if f, ok := metadata[e.Target()]; ok { return f.Do(context.Background(), e, from, until, values) } - return nil, fmt.Errorf("unknown function: %v", e.Target()) + return nil, errors.ErrUnknownFunction(e.Target()) }, } return e