Merge branch 'main' into feat/central-chart-repo

redpanda-data · Apr 10, 2024 · dc8d596 · dc8d596
2 parents 43ab63d + e3ce0e2
commit dc8d596
Show file tree

Hide file tree

Showing 13 changed files with 173 additions and 208 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -1,6 +1,9 @@
 name: Test chart
 
 on:
+  push:
+    branches:
+      - main
   workflow_call:
   workflow_dispatch:
   pull_request:

diff --git a/Chart.yaml b/Chart.yaml
@@ -15,7 +15,9 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 2.1.0
+
+version: 2.2.0
+
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/README.md b/README.md
@@ -1,98 +1,27 @@
-[![Chart status](https://img.shields.io/badge/Chart%20status-WIP-yellow)](https://github.com/benthosdev/benthos-helm-chart)
-[![benthos](https://img.shields.io/badge/benthos-v4.11.0-green)](https://github.com/Jeffail/benthos/releases/tag/v4.11.0)
-[![Chart version](https://img.shields.io/badge/Chart%20version-v0.8.0-green)](https://github.com/benthosdev/benthos-helm-chart/releases/tag/0.8.0)
-
 # benthos-helm-chart
+<p align="center" style="text-align: center">
+    <img src="./assets/blob.png" width="30%"><br/>
+    Benthos Helm Chart. <br/>
+</p>
 
-This chart is relatively immature, and may not yet support all workloads or benthos features.  Please feel free to test and provide feedback, report bugs, and/or contribute!
+Benthos is a high performance and resilient stream processor, able to connect various sources and sinks in a range of brokering patterns and perform hydration, enrichments, transformations and filters on payloads.
 
-## Repository
+This Helm Chart deploys a single Benthos instance in either streams mode or standalone.
 
-To add this repo:
-```
+## Installation
+```bash
 helm repo add benthos https://benthosdev.github.io/benthos-helm-chart/
-```
-Then `helm search repo benthos` for all charts.
+helm repo update
 
-### Versions
-Benthos is currently on v4.  If you need v3 for any reason, please set `image.tag` in your values.yaml.
+helm install benthos/benthos
+```
 
 ## Configuration
 
-### Benthos Parameters
-
-For more information on configuring the HTTP component, refer to the [Benthos HTTP component documentation](https://www.benthos.dev/docs/components/http/about).
-| Name                                          | Description                                      | Value              |
-|-----------------------------------------------|--------------------------------------------------|--------------------|
-| image.repository                              | Docker image repository                          | benthosdev/benthos |
-| image.pullPolicy                              | Docker image pull policy                         | IfNotPresent       |
-| image.tag                                     | Docker image tag override                        | ""                 |
-| imagePullSecrets                              | Docker registry secrets array                    | []                 |
-| nameOverride                                  | Sets name override                               | ""                 |
-| fullnameOverride                              | Sets full name override                          | ""                 |
-| serviceaccount.create                         | Enables creation of serviceaccount               | false              |
-| serviceaccount.annotations                    | Sets serviceaccount annotations                  | {}                 |
-| serviceaccount.name                           | Sets serviceaccount name                         | ""                 |
-| deployment.podAnnotations                     | Sets pod annotations                             | {}                 |
-| deployment.podLabels                          | Sets pod labels                                  | {}                 |
-| deployment.annotations                        | Set annotations on the Benthos Deployment        | {}                 |
-| deployment.terminationGracePeriodSeconds      | Override Benthos terminationGracePeriodSeconds.  | 60                 |
-| commonLabels                                  | Add labels that are common for all k8s components| ""                 |
-| podSecurityContext                            | Sets pod security context                        | {}                 |
-| command                                       | Replaces entrypoint command of benthos docker    | ""                 |
-| args                                          | Override default arguments passed with `command` | ""                 |
-| securityContext                               | Sets security context                            | {}                 |
-| service.type                                  | Kubernetes service type                          | ClusterIP          |
-| service.ports                                 | Kubernetes service ports                         | []                 |
-| ingress.enabled                               | Enables ingress                                  | false              |
-| ingress.className                             | Sets ingress class name                          | ""                 |
-| ingress.annotations                           | Sets ingress annotations                         | {}                 |
-| ingress.tls                                   | Sets ingress TLS configuration                   | []                 |
-| ingress.hosts                                 | Sets ingress hosts configuration                 | []                 |
-| env                                           | Sets benthos environment variables               | []                 |
-| envFrom                                       | Define env variables from Secrets or ConfigMaps  | []                 |
-| updateStrategy                                | Add Deployment Strategy                          | {}                 |
-| resources                                     | Set pod resource limits and/or requests          | {}                 |
-| autoscaling.enabled                           | Enables the horizontal pod autoscaler            | false              |
-| autoscaling.minReplicas                       | Sets min number of replicas                      | 1                  |
-| autoscaling.maxReplicas                       | Sets max numbers of replicas                     | 100                |
-| autoscaling.targetCPUUtilizationPercentage    | Sets desired CPU autoscaling threshold           | 80                 |
-| autoscaling.targetMemoryUtilizationPercentage | Sets desired memory autoscaling threshold        | 80                 |
-| autoscaling.behavior                          | Custom scale-up and scale-down behaviors         | {}                 |
-| autoscaling.customMetric                      | Target a custom metric for autoscaling           | {}                 |
-| nodeSelector                                  | Sets node selector configuration                 | {}                 |
-| tolerations                                   | Sets tolerations configuration                   | []                 |
-| affinity                                      | Sets affinity configuration                      | {}                 |
-| extraVolumes                                  | Defines extra volumes configuration              | []                 |
-| extraVolumeMounts                             | Sets additional mounts defined in extraVolumes   | []                 |
-| streams.enabled                               | Enables 'Benthos streams' mode                   | false              |
-| streams.streamsConfigMap                      | Name of K8s configMap containing streams configs | ""                 |
-| streams.api.enable                            | Enables streams API                              | true               |
-| http.enabled                                  | Enables the HTTP server component                | true               |
-| http.address                                  | HTTP server component binding address            | 0.0.0.0:4195       |
-| http.rootPath                                 | General Benthos HTTP endpoint prefix             | /benthos           |
-| http.debugEndpoints                           | Enables debugging endpoints                      | false              |
-| http.cors.enabled                             | Enables Cross-Origin Resource Sharing            | false              |
-| http.cors.allowedOrigins                      | Allowed source domains for CORS                  | ""                 |
-| http.tls.enabled                              | Enables TLS for all Benthos endpoints            | false              |
-| http.tls.secretName                           | `kubernetes.io/tls` secret name                  | ""                 |
-| podDisruptionBudget.enabled                   | Enables Pod Disruption Budget for Benthos        | false              |
-| podDisruptionBudget.minAvailable              | Min number of pods available at all times        | ""                 |
-| podDisruptionBudget.maxUnavailable            | Max unavailable number of pods                   | ""                 |
-| watch                                         | Enables watch mode                               | false              |
-| topologySpreadConstraints                     | Configure topology spread constraints            | []                 |
-| initContainers                                | Add any custom init container                    | []                 |
-| config                                        | Benthos component configuration                  | ""                 |
-
-## Config
-
 The config parameter should contain the configuration as it would be parsed by the Benthos binary.
-
-For example, the default Helm chart config block looks like this:
-
 ```yaml
-# /benthos.yaml configuration
-config: |-
+# values.yaml
+config:
   input:
     label: "no_config_in"
     generate:
@@ -104,19 +33,11 @@ config: |-
       codec: lines
 ```
 
-Adding an `http` block here is not recommended, please use the Helm directives described above.
-
-## TLS
-
-Benthos can be instructed to serve all endpoints exlusively over HTTPS.  This means that TLS configured in this way is not terminated at an ingress controller, but handled "end-to-end" at the container/binary. Prerequisites to enable TLS:
-- Set `service.port` to 443 in values.yaml
-- Create a Kubernetes secret in the targeted namespace of type `kubernetes.io/tls`
-
-When TLS is enabled, the Kubernetes readiness and liveness probes will operate over HTTPS to the same container port (default 4195).
+The full list of [available configuration for the Helm Chart can be found in the `values.yaml` file](./values.yaml). You should refer to the [upstream Benthos documentation](https://www.benthos.dev/docs/configuration/about) for the configuration of your pipeline.
 
 ## Streams mode
 
-When running Benthos in [streams mode](https://www.benthos.dev/docs/guides/streams_mode/about), all individual stream configuration files should be combined and placed in a single Kubernetes configMap, like so:
+When running Benthos in [streams mode](https://www.benthos.dev/docs/guides/streams_mode/about), all individual stream configuration files should be combined and placed in a single Kubernetes `ConfigMap`, like so:
 
 ```yaml
 apiVersion: v1
@@ -143,18 +64,35 @@ data:
       stdout:
         codec: lines
 ```
-Setting `streams.enabled: true` in `values.yaml` adds `streams /streams/*.yaml` to the pod command arguments.  
 
-This exposes a [streams API](https://www.benthos.dev/docs/guides/streams_mode/streams_api) by default, where configuration can be viewed and altered by anyone with access to the configured benthos endpoint.  As of benthos version `3.61.0`, the streams API can be disabled to prevent changes to the configuration[s] of a running [Benthos in streams mode via config files](https://www.benthos.dev/docs/guides/streams_mode/using_config_files); use `.Values.streams.api.enabled = false` to toggle the API off (this adds an additional argument to the pod command to become `streams --no-api /streams/*.yaml`).
+Then you can simply reference your `ConfigMap` and enable streams mode in your `values.yaml` file.
+```yaml
+# values.yaml
+streams:
+  enabled: true
+  streamsConfigMap: "benthos-streams"
+```
+
+Currently the streams mode `ConfigMap` should be applied **separately from and before installation of** the helm chart; support for deploying additional `ConfigMap`'s within the chart may be implemented later.
 
-With streams mode enabled, `/benthos.yaml` should only contain the configuration for `http` as defined either by the `http` values block or the defaults; the contents of the streams configMap (as defined and pulled into the deployment by the name set in `streams.streamsConfigMap`) are deployed to `/streams/*.yaml`. Additionally, general config like `metrics`, `logger `, and `tracing`.
+### Global Configuration
 
-From the `benthos streams --help` context:
+When deploying Benthos in streams mode, you may want to configure global tracing, logging and http configuration which is shared across all of your pipelines.
 
-```
-In streams mode the stream fields of a root target config (input, buffer,
-      pipeline, output) will be ignored. Other fields will be shared across all
-      loaded streams (resources, metrics, etc).
-```
+This can be done by specifying configuration under the `metrics`, `logger` and `tracing` configuration sections in your `values.yaml` file. These all use their respective upstream Benthos configuration syntax.
 
-Currently this streams configMap should be applied **separately from and before installation of** the helm chart; support for deploying additional configMaps within the chart may be implemented later.
+```yaml
+metrics:
+  prometheus: {}
+
+tracing:
+  openTelemetry:
+    http: []
+    grpc: []
+    tags: {}
+
+logger:
+  level: INFO
+  static_fields:
+    '@service': benthos
+```
diff --git a/assets/blob.png b/assets/blob.png
diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
@@ -63,14 +63,3 @@ Create the name of the service account to use
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}
 {{- end }}
-
-{{/*
-Return the appropriate apiVersion for HPA autoscaling APIs.
-*/}}
-{{- define "autoscaling.apiVersion" -}}
-{{- if or (.Capabilities.APIVersions.Has "autoscaling/v2/HorizontalPodAutoscaler") (semverCompare ">=1.23" .Capabilities.KubeVersion.Version) -}}
-autoscaling/v2
-{{- else -}}
-autoscaling/v2beta2
-{{- end -}}
-{{- end -}}
diff --git a/templates/configmap.yaml b/templates/configmap.yaml
@@ -24,9 +24,15 @@ data:
       {{- toYaml . | nindent 6}}
     {{- end }}
     {{- if and .Values.config (ne .Values.streams.enabled true) }}
-    {{- with .Values.config }}
-    {{- toYaml . | trimPrefix "|" | nindent 4}}
-    {{- end }}
+
+    {{- if kindIs "string" .Values.config }}
+    {{- tpl .Values.config . | nindent 4}}
+    {{- else if kindIs "map[string]interface{}" .Values.config }}
+    {{- toYaml (.Values.config | fromYaml) | nindent 4}}
+    {{- else }}
+    {{- toYaml .Values.config | nindent 4}}
+    {{- end -}}
+
     {{- else if and .Values.streams.enabled .Values.streams.streamsConfigMap }}
     {{- else }}
     input:

diff --git a/templates/deployment.yaml b/templates/deployment.yaml
@@ -37,6 +37,7 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       serviceAccountName: {{ include "benthos.serviceAccountName" . }}
+      restartPolicy: {{ .Values.deployment.restartPolicy }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       terminationGracePeriodSeconds: {{ default 60 .Values.deployment.terminationGracePeriodSeconds }}
@@ -71,7 +72,7 @@ spec:
             {{- end }}
             - /streams/*.yaml
             {{- end }}
-          {{- end }}  
+          {{- end }}
           ports:
             - name: http
               {{- if .Values.http.address}}
@@ -82,21 +83,9 @@ spec:
               protocol: TCP
           {{- if .Values.http.enabled }}
           livenessProbe:
-            httpGet:
-              path: /ping
-              port: http
-              {{- if and .Values.http.cert_file .Values.http.key_file}}
-              scheme: HTTPS
-              {{- end }}
-              {{- toYaml .Values.deployment.livenessProbe | nindent 14 }}
+            {{- toYaml .Values.deployment.livenessProbe | nindent 12 }}
           readinessProbe:
-            httpGet:
-              path: /ready
-              port: http
-              {{- if and .Values.http.cert_file .Values.http.key_file}}
-              scheme: HTTPS
-              {{- end }}
-              {{- toYaml .Values.deployment.readinessProbe | nindent 14 }}
+            {{- toYaml .Values.deployment.readinessProbe | nindent 12 }}
           {{- end }}
           env:
             {{- toYaml .Values.env | nindent 12 }}

diff --git a/templates/hpa.yaml b/templates/hpa.yaml
@@ -1,5 +1,5 @@
 {{- if .Values.autoscaling.enabled }}
-apiVersion: {{ template "autoscaling.apiVersion" . }}
+apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:
   name: {{ include "benthos.fullname" . }}

diff --git a/templates/servicemonitor.yaml b/templates/servicemonitor.yaml
@@ -6,15 +6,19 @@ metadata:
   labels:
     {{- include "benthos.labels" . | nindent 4 }}
 spec:
-  {{- with .Values.serviceMonitor.basicAuth }}
-  basicAuth:
-      {{- toYaml . | nindent 4 }}
-  {{- end }}
   endpoints:
     - interval: {{ .Values.serviceMonitor.interval}}
       targetPort: http
       path: /metrics
       scheme: {{ .Values.serviceMonitor.scheme }}
+      {{- with .Values.serviceMonitor.basicAuth }}
+      basicAuth:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.serviceMonitor.tlsConfig }}
+      tlsConfig:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
   namespaceSelector:
     matchNames:
       - {{ .Release.Namespace }}

diff --git a/tests/__snapshot__/configuration_test.yaml.snap b/tests/__snapshot__/configuration_test.yaml.snap
@@ -34,46 +34,45 @@ can set global options with streams mode:
         root_path: /benthos
 should set config from string:
   1: |
-    benthos.yaml: |-
+    benthos.yaml: |
       http:
         address: 0.0.0.0:4195
         cors:
           enabled: false
         debug_endpoints: false
         enabled: true
         root_path: /benthos
+      input:
+        nats_jetstream:
+          urls: [ nats://TODO:4222 ]
+          queue: myqueue
+          subject: traffic.light.events
+          deliver: all
 
-        input:
-          nats_jetstream:
-            urls: [ nats://TODO:4222 ]
-            queue: myqueue
-            subject: traffic.light.events
-            deliver: all
-
-        buffer:
-          system_window:
-            timestamp_mapping: root = this.created_at
-            size: 1h
+      buffer:
+        system_window:
+          timestamp_mapping: root = this.created_at
+          size: 1h
 
-        pipeline:
-          processors:
-            - group_by_value:
-                value: '${! json("traffic_light_id") }'
-            - mapping: |
-                root = if batch_index() == 0 {
-                  {
-                    "traffic_light_id": this.traffic_light_id,
-                    "created_at": @window_end_timestamp,
-                    "total_cars": json("registration_plate").from_all().unique().length(),
-                    "passengers": json("passengers").from_all().sum(),
-                  }
-                } else { deleted() }
+      pipeline:
+        processors:
+          - group_by_value:
+              value: '${! json("traffic_light_id") }'
+          - mapping: |
+              root = if batch_index() == 0 {
+                {
+                  "traffic_light_id": this.traffic_light_id,
+                  "created_at": @window_end_timestamp,
+                  "total_cars": json("registration_plate").from_all().unique().length(),
+                  "passengers": json("passengers").from_all().sum(),
+                }
+              } else { deleted() }
 
-        output:
-          http_client:
-            url: https://example.com/traffic_data
-            verb: POST
-            max_in_flight: 64
+      output:
+        http_client:
+          url: https://example.com/traffic_data
+          verb: POST
+          max_in_flight: 64
 should set config from yaml:
   1: |
     benthos.yaml: |-