Skip to content

Commit dcc07a3

Browse files
authored
Introduce common helm library chart (#386)
Introduces a new `common-app` library chart to de-duplicate all common templates and logic (deployment, service, helpers, etc.). This simplifies maintenance and ensures consistency across all applications, especially once more applications get added in the future. All application charts (`streams-app`, `producer-app`, and cleanup jobs) now depend on `common-app`. Application templates are simplified to `include` logic from the library.
1 parent b8c55d4 commit dcc07a3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+776
-799
lines changed

.github/workflows/build-and-publish.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ on:
1010
jobs:
1111
build-and-publish:
1212
name: Java Gradle
13-
uses: bakdata/ci-templates/.github/workflows/[email protected].0
13+
uses: bakdata/ci-templates/.github/workflows/[email protected].2
1414
with:
1515
java-version: 17
1616
secrets:

.github/workflows/helm-lint.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ jobs:
88
runs-on: ubuntu-22.04
99
steps:
1010
- name: Lint Helm chart
11-
uses: bakdata/ci-templates/actions/[email protected].0
11+
uses: bakdata/ci-templates/actions/[email protected].2
1212
with:
1313
lint-config-path: ".github/lint-config.yaml"
1414
ref: ${{ github.ref_name }}

.github/workflows/helm-publish.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@ jobs:
1111
version: ${{ steps.get-version.outputs.version }}
1212
steps:
1313
- name: Check out repository
14-
uses: bakdata/ci-templates/actions/[email protected].0
14+
uses: bakdata/ci-templates/actions/[email protected].2
1515

1616
- name: Set up Gradle with version ${{ inputs.gradle-version }}
17-
uses: bakdata/ci-templates/actions/[email protected].0
17+
uses: bakdata/ci-templates/actions/[email protected].2
1818
with:
1919
java-distribution: "microsoft"
2020
java-version: "11"
@@ -32,7 +32,7 @@ jobs:
3232

3333
helm-publish:
3434
name: Publish Helm chart
35-
uses: bakdata/ci-templates/.github/workflows/[email protected].0
35+
uses: bakdata/ci-templates/.github/workflows/[email protected].2
3636
needs: get-gradle-version
3737
with:
3838
charts-path: "./charts"

.github/workflows/release.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ on:
1616
jobs:
1717
java-gradle-release:
1818
name: Java Gradle
19-
uses: bakdata/ci-templates/.github/workflows/[email protected].0
19+
uses: bakdata/ci-templates/.github/workflows/[email protected].2
2020
with:
2121
java-version: 17
2222
release-type: "${{ inputs.release-type }}"

charts/common-app/Chart.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: v2
2+
name: common-app
3+
description: A common library chart for Kafka applications.
4+
type: library
5+
version: 0.1.0
6+
appVersion: "0.1.0"
7+
maintainers:
8+
- name: bakdata
9+
10+
url: bakdata.com

charts/common-app/README.md

Lines changed: 212 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{{- define "common-app.configmap" -}}
2+
{{ if .Values.files }}
3+
apiVersion: v1
4+
kind: ConfigMap
5+
metadata:
6+
name: {{ include "common-app.fullname" . }}
7+
{{- include "common-app.annotations" . }}
8+
labels:
9+
{{- include "common-app.labels" . | nindent 4 }}
10+
{{- range $key, $value := .Values.labels }}
11+
{{ $key | quote }}: {{ $value | quote }}
12+
{{- end }}
13+
data:
14+
{{- range $key, $value := .Values.files }}
15+
{{ $key }}: {{ $value.content | quote }}
16+
{{- end }}
17+
{{ end }}
18+
{{- end -}}
Lines changed: 278 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,278 @@
1+
{{- define "common-app.deployment" -}}
2+
{{- $root := . -}}
3+
{{- if and .Values.persistence.enabled (not .Values.persistence.size) }}
4+
{{- fail "When persistence is enabled, you must set .Values.persistence.size" }}
5+
{{- end }}
6+
7+
{{- if .Capabilities.APIVersions.Has "apps/v1" }}
8+
apiVersion: apps/v1
9+
{{- else }}
10+
apiVersion: apps/v1beta1
11+
{{- end }}
12+
{{- if .Values.statefulSet }}
13+
kind: StatefulSet
14+
{{- else }}
15+
kind: Deployment
16+
{{- end }}
17+
metadata:
18+
name: {{ include "common-app.fullname" . }}
19+
{{- include "common-app.deployment-annotations" . }}
20+
labels:
21+
{{- include "common-app.labels" . | nindent 4 }}
22+
streams-bootstrap/kind: {{ .Chart.Name }}
23+
{{- range $key, $value := .Values.labels }}
24+
{{ $key | quote }}: {{ $value | quote }}
25+
{{- end }}
26+
spec:
27+
{{- if .Values.statefulSet }}
28+
serviceName: {{ include "common-app.fullname" . }}
29+
podManagementPolicy: Parallel
30+
{{- end }}
31+
{{- if (not .Values.autoscaling.enabled) }}
32+
replicas: {{ .Values.replicaCount }}
33+
{{- end }}
34+
selector:
35+
matchLabels:
36+
{{- include "common-app.selectorLabels" . | nindent 6 }}
37+
template:
38+
metadata:
39+
{{- if or .Values.podAnnotations .Values.files }}
40+
annotations:
41+
{{- if .Values.files }}
42+
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
43+
{{- end }}
44+
{{- range $key, $value := .Values.podAnnotations }}
45+
{{ $key | quote }}: {{ $value | quote }}
46+
{{- end }}
47+
{{- end }}
48+
labels:
49+
{{- include "common-app.selectorLabels" . | nindent 8 }}
50+
streams-bootstrap/kind: {{ .Chart.Name }}
51+
{{- range $key, $value := .Values.podLabels }}
52+
{{ $key }}: {{ $value }}
53+
{{- end }}
54+
spec:
55+
{{- if .Values.serviceAccountName }}
56+
serviceAccountName: {{ .Values.serviceAccountName }}
57+
{{- end }}
58+
{{- if .Values.tolerations }}
59+
tolerations:
60+
{{ toYaml .Values.tolerations | indent 8 }}
61+
{{- end }}
62+
{{- with .Values.affinity }}
63+
affinity:
64+
{{- tpl (toYaml .) $root | nindent 8 }}
65+
{{- end }}
66+
{{- if .Values.priorityClassName }}
67+
priorityClassName: {{ .Values.priorityClassName }}
68+
{{- end }}
69+
terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
70+
{{- if .Values.imagePullSecrets }}
71+
imagePullSecrets:
72+
{{- toYaml .Values.imagePullSecrets | nindent 8 }}
73+
{{- end }}
74+
containers:
75+
- name: "kafka-app"
76+
image: "{{ .Values.image }}:{{ .Values.imageTag }}"
77+
imagePullPolicy: "{{ .Values.imagePullPolicy }}"
78+
resources:
79+
{{ toYaml .Values.resources | indent 12 }}
80+
env:
81+
- name: ENV_PREFIX
82+
value: {{ .Values.configurationEnvPrefix }}_
83+
{{- range $key, $value := .Values.kafka.config }}
84+
- name: {{ printf "KAFKA_%s" $key | replace "." "_" | upper | quote }}
85+
value: {{ $value | quote }}
86+
{{- end }}
87+
{{- range .Values.ports }}
88+
{{- if .servicePort }} # TODO verify that there is at most one service port. Currently, if there are multiple service ports, the first one will be used
89+
- name: POD_IP
90+
valueFrom:
91+
fieldRef:
92+
fieldPath: status.podIP
93+
- name: KAFKA_APPLICATION_SERVER
94+
value: "$(POD_IP):{{ .containerPort }}"
95+
{{- end }}
96+
{{- end }}
97+
{{- if .Values.kafka.staticMembership }}
98+
- name: KAFKA_GROUP_INSTANCE_ID
99+
valueFrom:
100+
fieldRef:
101+
fieldPath: metadata.name
102+
{{- end }}
103+
{{- if not .Values.statefulSet }}
104+
- name: "{{ .Values.configurationEnvPrefix }}_VOLATILE_GROUP_INSTANCE_ID"
105+
value: "true"
106+
{{- end }}
107+
{{- if hasKey .Values.kafka "bootstrapServers" }}
108+
- name: "{{ .Values.configurationEnvPrefix }}_BOOTSTRAP_SERVERS"
109+
value: {{ .Values.kafka.bootstrapServers | quote }}
110+
{{- end }}
111+
{{- if hasKey .Values.kafka "schemaRegistryUrl" }}
112+
- name: "{{ .Values.configurationEnvPrefix }}_SCHEMA_REGISTRY_URL"
113+
value: {{ .Values.kafka.schemaRegistryUrl | quote }}
114+
{{- end }}
115+
{{- if and (hasKey .Values.kafka "inputTopics") (.Values.kafka.inputTopics) }}
116+
- name: "{{ .Values.configurationEnvPrefix }}_INPUT_TOPICS"
117+
value: {{ .Values.kafka.inputTopics | join "," | quote }}
118+
{{- end }}
119+
{{- if hasKey .Values.kafka "inputPattern" }}
120+
- name: "{{ .Values.configurationEnvPrefix }}_INPUT_PATTERN"
121+
value: {{ .Values.kafka.inputPattern | quote }}
122+
{{- end }}
123+
{{- if hasKey .Values.kafka "outputTopic" }}
124+
- name: "{{ .Values.configurationEnvPrefix }}_OUTPUT_TOPIC"
125+
value: {{ .Values.kafka.outputTopic | quote }}
126+
{{- end }}
127+
{{- if hasKey .Values.kafka "errorTopic" }}
128+
- name: "{{ .Values.configurationEnvPrefix }}_ERROR_TOPIC"
129+
value: {{ .Values.kafka.errorTopic | quote }}
130+
{{- end }}
131+
{{- if and (hasKey .Values.kafka "labeledOutputTopics") (.Values.kafka.labeledOutputTopics) }}
132+
- name: "{{ .Values.configurationEnvPrefix }}_LABELED_OUTPUT_TOPICS"
133+
value: "{{- range $key, $value := .Values.kafka.labeledOutputTopics }}{{ $key }}={{ $value }},{{- end }}"
134+
{{- end }}
135+
{{- $delimiter := ";" }}
136+
{{- if and (hasKey .Values.kafka "labeledInputTopics") (.Values.kafka.labeledInputTopics) }}
137+
- name: "{{ .Values.configurationEnvPrefix }}_LABELED_INPUT_TOPICS"
138+
value: "{{- range $key, $value := .Values.kafka.labeledInputTopics }}{{ $key }}={{ $value | join $delimiter }},{{- end }}"
139+
{{- end }}
140+
{{- if and (hasKey .Values.kafka "labeledInputPatterns") (.Values.kafka.labeledInputPatterns) }}
141+
- name: "{{ .Values.configurationEnvPrefix }}_LABELED_INPUT_PATTERNS"
142+
value: "{{- range $key, $value := .Values.kafka.labeledInputPatterns }}{{ $key }}={{ $value }},{{- end }}"
143+
{{- end }}
144+
{{- if hasKey .Values.kafka "applicationId" }}
145+
- name: "{{ .Values.configurationEnvPrefix }}_APPLICATION_ID"
146+
value: {{ .Values.kafka.applicationId | quote }}
147+
{{- end }}
148+
{{- range $key, $value := .Values.secrets }}
149+
- name: "{{ $key }}"
150+
valueFrom:
151+
secretKeyRef:
152+
name: {{ include "common-app.fullname" . }}
153+
key: "{{ $key }}"
154+
{{- end }}
155+
{{- range $key, $value := .Values.secretRefs }}
156+
- name: "{{ $key }}"
157+
valueFrom:
158+
secretKeyRef:
159+
name: {{ $value.name }}
160+
key: "{{ $value.key }}"
161+
{{- end }}
162+
{{- range $key, $value := .Values.commandLine }}
163+
- name: "{{ $root.Values.configurationEnvPrefix }}_{{ $key }}"
164+
value: {{ $value | quote }}
165+
{{- end }}
166+
{{- range $key, $value := .Values.env }}
167+
- name: {{ $key | quote }}
168+
value: {{ $value | quote }}
169+
{{- end }}
170+
- name: JAVA_TOOL_OPTIONS
171+
value: '-Dcom.sun.management.jmxremote.port={{ .Values.jmx.port }}
172+
-Dcom.sun.management.jmxremote.authenticate=false
173+
-Dcom.sun.management.jmxremote.ssl=false
174+
{{- if .Values.jmx.enabled }}
175+
-Djava.rmi.server.hostname={{ .Values.jmx.host }}
176+
-Dcom.sun.management.jmxremote.rmi.port={{ .Values.jmx.port }}
177+
{{- end }}
178+
-XX:MaxRAMPercentage={{ printf "%.1f" .Values.javaOptions.maxRAMPercentage }}
179+
{{ .Values.javaOptions.others | join " " }}'
180+
{{- if or (.Values.files) (and .Values.persistence.enabled .Values.statefulSet) (.Values.secretFilesRefs) }}
181+
volumeMounts:
182+
{{- range $key, $value := .Values.files }}
183+
- name: config
184+
mountPath: {{ printf "%s/%s" $value.mountPath $key | quote }}
185+
subPath: {{ $key | quote }}
186+
{{- end }}
187+
{{- range .Values.secretFilesRefs }}
188+
- name: {{ .volume }}
189+
mountPath: {{ .mountPath }}
190+
{{- if .readOnly }}
191+
readOnly: true
192+
{{- end }}
193+
{{- if .subPath}}
194+
subPath: {{.subPath }}
195+
{{- end }}
196+
{{- end }}
197+
{{- if and .Values.persistence.enabled .Values.statefulSet }}
198+
- name: datadir
199+
mountPath: /tmp/kafka-streams
200+
{{- end }}
201+
{{- end }}
202+
{{- if or (.Values.jmx.enabled) (.Values.ports) }}
203+
ports:
204+
{{- range .Values.ports }}
205+
- containerPort: {{ .containerPort }}
206+
name: {{ .name | quote }}
207+
{{- if .protocol }}
208+
protocol: {{ .protocol | quote }}
209+
{{- end }}
210+
{{- end }}
211+
{{- if .Values.jmx.enabled }}
212+
- containerPort: {{ .Values.jmx.port }}
213+
name: jmx
214+
{{- end }}
215+
{{- end }}
216+
{{- if .Values.livenessProbe }}
217+
livenessProbe:
218+
{{- .Values.livenessProbe | toYaml | nindent 12 }}
219+
{{- end }}
220+
{{- if .Values.readinessProbe }}
221+
readinessProbe:
222+
{{- .Values.readinessProbe | toYaml | nindent 12 }}
223+
{{- end }}
224+
{{- if .Values.prometheus.jmx.enabled }}
225+
- name: prometheus-jmx-exporter
226+
image: "{{ .Values.prometheus.jmx.image }}:{{ .Values.prometheus.jmx.imageTag }}"
227+
imagePullPolicy: "{{ .Values.prometheus.jmx.imagePullPolicy }}"
228+
args:
229+
- {{ .Values.prometheus.jmx.port | quote }}
230+
- /etc/jmx-{{ include "common-app.name" . }}/jmx-kafka-{{ include "common-app.name" . }}-prometheus.yml
231+
env:
232+
- name: JAVA_TOOL_OPTIONS
233+
value: "-XX:MaxRAMPercentage=90.0"
234+
ports:
235+
- containerPort: {{ .Values.prometheus.jmx.port }}
236+
name: prometheus
237+
resources:
238+
{{ toYaml .Values.prometheus.jmx.resources | indent 12 }}
239+
volumeMounts:
240+
- name: jmx-config
241+
mountPath: /etc/jmx-{{ include "common-app.name" . }}
242+
{{- end }}
243+
{{- if or (.Values.prometheus.jmx.enabled) (.Values.files) (.Values.secretFilesRefs) }}
244+
volumes:
245+
{{- if .Values.prometheus.jmx.enabled }}
246+
- name: jmx-config
247+
configMap:
248+
name: {{ include "common-app.fullname" . }}-jmx
249+
{{- end }}
250+
{{- if .Values.files }}
251+
- name: config
252+
configMap:
253+
name: {{ include "common-app.fullname" . }}
254+
{{- end }}
255+
{{- range .Values.secretFilesRefs }}
256+
- name: {{ .volume }}
257+
secret:
258+
secretName: {{ .name }}
259+
{{- end }}
260+
{{- end }}
261+
{{- if and .Values.persistence.enabled .Values.statefulSet }}
262+
volumeClaimTemplates:
263+
- metadata:
264+
name: datadir
265+
spec:
266+
accessModes: [ "ReadWriteOnce" ]
267+
resources:
268+
requests:
269+
storage: "{{ .Values.persistence.size }}"
270+
{{- if .Values.persistence.storageClass }}
271+
{{- if (eq "-" .Values.persistence.storageClass) }}
272+
storageClassName: ""
273+
{{- else }}
274+
storageClassName: "{{ .Values.persistence.storageClass }}"
275+
{{- end }}
276+
{{- end }}
277+
{{- end }}
278+
{{- end -}}

0 commit comments

Comments
 (0)