diff --git a/build/debezium-fixer/Dockerfile b/build/debezium-fixer/Dockerfile index a5d4d8b..a6dba3e 100644 --- a/build/debezium-fixer/Dockerfile +++ b/build/debezium-fixer/Dockerfile @@ -1,5 +1,14 @@ FROM alpine:3.15.4 +ARG SOURCE +ARG COMMIT_HASH +ARG COMMIT_ID +ARG BUILD_TIME +LABEL source=${SOURCE} +LABEL commit_hash=${COMMIT_HASH} +LABEL commit_id=${COMMIT_ID} +LABEL build_time=${BUILD_TIME} + ARG container_user=mosip ARG container_user_group=mosip ARG container_user_uid=1001 diff --git a/build/es-kafka-connecter/kafka_connect_transforms/Dockerfile b/build/es-kafka-connecter/kafka_connect_transforms/Dockerfile index 57096bb..087dc0e 100644 --- a/build/es-kafka-connecter/kafka_connect_transforms/Dockerfile +++ b/build/es-kafka-connecter/kafka_connect_transforms/Dockerfile @@ -1,5 +1,14 @@ FROM confluentinc/cp-kafka-connect:7.0.0-1-ubi8 +ARG SOURCE +ARG COMMIT_HASH +ARG COMMIT_ID +ARG BUILD_TIME +LABEL source=${SOURCE} +LABEL commit_hash=${COMMIT_HASH} +LABEL commit_id=${COMMIT_ID} +LABEL build_time=${BUILD_TIME} + RUN confluent-hub install --no-prompt confluentinc/kafka-connect-elasticsearch:11.1.3 ADD ./target/kafka_connect_transforms-*.jar /usr/share/java/my_transforms diff --git a/build/es-kafka-connecter/kafka_connect_transforms/pom.xml b/build/es-kafka-connecter/kafka_connect_transforms/pom.xml index 53c79c8..e0e33ab 100644 --- a/build/es-kafka-connecter/kafka_connect_transforms/pom.xml +++ b/build/es-kafka-connecter/kafka_connect_transforms/pom.xml @@ -30,6 +30,7 @@ io.mosip kafka_connect_transforms kafka_connect_transforms + Mosip Kafka Connect Transform Project 1.2.1-SNAPSHOT https://github.com/mosip/reporting @@ -198,4 +199,4 @@ - \ No newline at end of file + diff --git a/build/reporting-init/debezium-init/Dockerfile b/build/reporting-init/debezium-init/Dockerfile index b109811..a00f8d1 100644 --- a/build/reporting-init/debezium-init/Dockerfile +++ b/build/reporting-init/debezium-init/Dockerfile @@ -1,5 +1,14 @@ FROM alpine +ARG SOURCE +ARG COMMIT_HASH +ARG COMMIT_ID +ARG BUILD_TIME +LABEL source=${SOURCE} +LABEL commit_hash=${COMMIT_HASH} +LABEL commit_id=${COMMIT_ID} +LABEL build_time=${BUILD_TIME} + RUN apk add curl bash vim ADD docker-entrypoint.sh / diff --git a/build/reporting-init/es-kafka-connect-init/Dockerfile b/build/reporting-init/es-kafka-connect-init/Dockerfile index c225f5f..1e84d18 100644 --- a/build/reporting-init/es-kafka-connect-init/Dockerfile +++ b/build/reporting-init/es-kafka-connect-init/Dockerfile @@ -1,5 +1,14 @@ FROM alpine +ARG SOURCE +ARG COMMIT_HASH +ARG COMMIT_ID +ARG BUILD_TIME +LABEL source=${SOURCE} +LABEL commit_hash=${COMMIT_HASH} +LABEL commit_id=${COMMIT_ID} +LABEL build_time=${BUILD_TIME} + RUN apk add curl bash vim RUN mkdir /connectors /kibana_saved_objects diff --git a/helm/reporting-init/.gitignore b/helm/reporting-init/.gitignore new file mode 100644 index 0000000..ee3892e --- /dev/null +++ b/helm/reporting-init/.gitignore @@ -0,0 +1 @@ +charts/ diff --git a/helm/reporting-init/.helmignore b/helm/reporting-init/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/helm/reporting-init/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/reporting-init/Chart.yaml b/helm/reporting-init/Chart.yaml new file mode 100644 index 0000000..490aaf3 --- /dev/null +++ b/helm/reporting-init/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: reporting-init +description: A Helm chart for MOSIP Reporting Initialization. +type: application +version: 0.0.1-develop +appVersion: "" +maintainers: + - email: info@mosip.io + name: MOSIP diff --git a/helm/reporting-init/templates/debezium-job.yaml b/helm/reporting-init/templates/debezium-job.yaml new file mode 100644 index 0000000..b7e39e1 --- /dev/null +++ b/helm/reporting-init/templates/debezium-job.yaml @@ -0,0 +1,64 @@ +{{ if .Values.debezium_connectors.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-debezium-job + labels: + app.kubernetes.io/name: {{ .Release.Name }}-debezium-job +spec: + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Release.Name }}-debezium-job + sidecar.istio.io/inject: "false" + spec: + restartPolicy: Never # This is one time job + containers: + - name: debezium-init + image: "{{ .Values.debezium_connectors.image }}:{{ .Values.debezium_connectors.imageTag }}" + imagePullPolicy: {{ .Values.debezium_connectors.imagePullPolicy }} + env: + - name: DB_PREFIX_INDEX + value: {{ .Values.base.db_prefix }} + - name: DB_HOSTNAME + value: {{ .Values.base.db_hostname }} + - name: DB_PORT + value: {{ .Values.base.db_port | quote }} + - name: DB_USER + value: {{ .Values.base.db_user }} + - name: DB_PASS + {{ if .Values.base.db_pass_direct }} + value: {{ .Values.base.db_pass_direct }} + {{ else }} + valueFrom: + secretKeyRef: + name: {{ .Values.base.db_pass_secretref.name }} + key: {{ .Values.base.db_pass_secretref.key }} + {{ end }} + - name: ES_URL + value: {{ .Values.base.es_url }} + - name: KIBANA_URL + value: {{ .Values.base.kibana_url }} + + - name: NUMBER_DEBEZ_CONN + value: {{ len .Values.debezium_connectors.dbs | quote }} + - name: DELAY_BETWEEN_CONNECTORS + value: {{ .Values.debezium_connectors.delay_between_connectors | quote }} + {{- range $i, $item := .Values.debezium_connectors.dbs }} + - name: DEBEZ_DB_NAME_{{ add $i 1 }} + value: {{ $item.name | quote }} + - name: DEBEZ_TABLE_LIST_{{ add $i 1 }} + value: {{ $item.tables | quote }} + {{- end }} + volumeMounts: + - name: debez-sample-file + mountPath: "/debez-connector" + volumes: + - name: debez-sample-file + configMap: + {{ if .Values.debezium_connectors.existingConfigMap }} + name: {{ .Values.debezium_connectors.existingConfigMap }} + {{ else }} + name: {{ .Release.Name }}-debezium-sample-conf + {{ end }} +{{ end }} diff --git a/helm/reporting-init/templates/es-kafka-connect-job.yaml b/helm/reporting-init/templates/es-kafka-connect-job.yaml new file mode 100644 index 0000000..9d4a698 --- /dev/null +++ b/helm/reporting-init/templates/es-kafka-connect-job.yaml @@ -0,0 +1,56 @@ +{{ if .Values.es_kafka_connectors.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-es-kafka-conn-job + labels: + app.kubernetes.io/name: {{ .Release.Name }}-es-kafka-conn-job +spec: + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Release.Name }}-es-kafka-conn-job + sidecar.istio.io/inject: "false" + spec: + restartPolicy: Never # This is one time job + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.es_kafka_connectors.image }}:{{ .Values.es_kafka_connectors.imageTag }}" + imagePullPolicy: {{ .Values.es_kafka_connectors.imagePullPolicy }} + env: + - name: DB_PREFIX_INDEX + value: {{ .Values.base.db_prefix }} + - name: DB_HOSTNAME + value: {{ .Values.base.db_hostname }} + - name: DB_PORT + value: {{ .Values.base.db_port | quote }} + - name: DB_USER + value: {{ .Values.base.db_user }} + - name: DB_PASS + {{ if .Values.base.db_pass_direct }} + value: {{ .Values.base.db_pass_direct }} + {{ else }} + valueFrom: + secretKeyRef: + name: {{ .Values.base.db_pass_secretref.name }} + key: {{ .Values.base.db_pass_secretref.key }} + {{ end }} + - name: ES_URL + value: {{ .Values.base.es_url }} + - name: KIBANA_URL + value: {{ .Values.base.kibana_url }} + + - name: DELAY_BETWEEN_CONNECTORS + value: {{ .Values.es_kafka_connectors.delay_between_connectors | quote }} + volumeMounts: + - name: {{ .Release.Name }}-es-kafka-connectors + mountPath: "/connectors/{{ .Release.Name }}-es-kafka-connectors" + volumes: + - name: {{ .Release.Name }}-es-kafka-connectors + configMap: + {{ if .Values.es_kafka_connectors.existingConfigMap }} + name: {{ .Values.es_kafka_connectors.existingConfigMap }} + {{ else }} + name: {{ .Release.Name }}-es-kafka-connectors-conf + {{ end }} +{{ end }} diff --git a/helm/reporting-init/values.yaml b/helm/reporting-init/values.yaml new file mode 100644 index 0000000..63d4a54 --- /dev/null +++ b/helm/reporting-init/values.yaml @@ -0,0 +1,43 @@ +base: + db_prefix: "mosip" + db_user: postgres + db_hostname: "postgres-postgresql.postgres" + db_port: "5432" + # if this is not false it will directly take pass from this variable, else it will take from db_pass_secretref + db_pass_direct: false + db_pass_secretref: + name: postgres-postgresql + key: postgresql-password + es_url: http://elasticsearch-master.cattle-logging-system:9200 + kibana_url: http://kibana.cattle-logging-system:5601 + +debezium_connectors: + image: mosipqa/debezium-init + imageTag: develop + imagePullPolicy: Always + + enabled: true + existingConfigMap: null + delay_between_connectors: "0s" + dbs: + - name: "mosip_master" + tables: "master.doc_type,master.registration_center" + - name: "mosip_audit" + tables: "audit.app_audit_log" + - name: "mosip_prereg" + tables: "prereg.applicant_demographic,prereg.applicant_demographic_consumed,prereg.applicant_document,prereg.applicant_document_consumed,prereg.reg_appointment,prereg.reg_appointment_consumed,prereg.reg_available_slot,prereg.otp_transaction" + - name: "mosip_ida" + tables: "ida.auth_transaction" + - name: "mosip_regprc" + tables: "regprc.registration,regprc.registration_list" + +es_kafka_connectors: + image: mosipqa/es-kafka-connect-init + imageTag: develop + imagePullPolicy: Always + + enabled: true + existingConfigMap: null + delay_between_connectors: "0s" + +kibana_enabled: {} diff --git a/helm/reporting/.gitignore b/helm/reporting/.gitignore new file mode 100644 index 0000000..ee3892e --- /dev/null +++ b/helm/reporting/.gitignore @@ -0,0 +1 @@ +charts/ diff --git a/helm/reporting/.helmignore b/helm/reporting/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/helm/reporting/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/reporting/Chart.yaml b/helm/reporting/Chart.yaml new file mode 100644 index 0000000..232e03a --- /dev/null +++ b/helm/reporting/Chart.yaml @@ -0,0 +1,30 @@ +apiVersion: v2 +name: reporting +description: A Helm chart for MOSIP Reporting module. +type: application +version: 0.0.1-develop +appVersion: "" +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +- name: cp-helm-charts + alias: confluentinc + repository: "https://confluentinc.github.io/cp-helm-charts/" + version: ">= 0.6.1" + condition: confluentinc.enabled +- name: kafka + alias: bitnami-kafka + repository: "https://charts.bitnami.com/bitnami" + version: "23.0.7" + condition: bitnami-kafka.enabled +- name: elasticsearch + alias: bitnami-elasticsearch + repository: "https://charts.bitnami.com/bitnami" + version: ">= 17.0.0" + condition: bitnami-elasticsearch.enabled +maintainers: + - email: info@mosip.io + name: MOSIP diff --git a/helm/reporting/README.md b/helm/reporting/README.md new file mode 100644 index 0000000..7c704f1 --- /dev/null +++ b/helm/reporting/README.md @@ -0,0 +1,19 @@ +# Reporting Framework + +Helm chart for installing reporting module + +## TL;DR + +```console +$ helm repo add mosip https://mosip.github.io/mosip-helm +$ helm install my-release mosip/reporting +``` + +## Contents + +This helm chart contains the following subcharts and they can be individually configured/installed/omitted, through the `values.yaml`. +- Bitnami's Kafka +- Debezium Kafka Connector +- Elasticsearch Kafka Connector +- A KafkaClient Pod for monitoring kafka topics, and making connect api calls + diff --git a/helm/reporting/templates/debezium.yaml b/helm/reporting/templates/debezium.yaml new file mode 100644 index 0000000..3eb1882 --- /dev/null +++ b/helm/reporting/templates/debezium.yaml @@ -0,0 +1,86 @@ +{{- if .Values.debezium.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: debezium-deploy + labels: + app: debezium +spec: + replicas: {{ .Values.debezium.replicaCount }} + selector: + matchLabels: + app: debezium + template: + metadata: + labels: + app: debezium + spec: + containers: + - name: debezium-container + image: {{ .Values.debezium.image }} + imagePullPolicy: {{ .Values.debezium.imagePullPolicy }} + env: + - name: BOOTSTRAP_SERVERS + value: {{ .Values.debezium.kafka.server }} + - name: GROUP_ID + value: {{ .Values.debezium.kafka.groupId | quote }} + - name: OFFSET_STORAGE_TOPIC + value: {{ .Values.debezium.kafka.topics.offset }} + - name: CONFIG_STORAGE_TOPIC + value: {{ .Values.debezium.kafka.topics.config }} + - name: STATUS_STORAGE_TOPIC + value: {{ .Values.debezium.kafka.topics.status }} + ports: + - containerPort: {{ .Values.debezium.port }} + {{- if .Values.debezium.startupProbe.enabled }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.debezium.startupProbe "enabled") "context" $) | nindent 10 }} + {{- end }} + {{- if .Values.debezium.livenessProbe.enabled }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.debezium.livenessProbe "enabled") "context" $) | nindent 10 }} + {{- end }} + {{- if .Values.debezium.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.debezium.readinessProbe "enabled") "context" $) | nindent 10 }} + {{- end }} + - name: debezium-fixer + image: {{ .Values.debezium.fixer_container.image }} + imagePullPolicy: {{ .Values.debezium.fixer_container.imagePullPolicy }} + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: STARTUP_DELAY + value: {{ .Values.debezium.fixer_container.startup_delay }} + - name: PROBE_TIME_PERIOD + value: {{ .Values.debezium.fixer_container.probe_time_period }} + - name: ERROR_LEVEL1 + value: {{ .Values.debezium.fixer_container.error_level1 | quote }} + - name: ERROR_LEVEL2 + value: {{ .Values.debezium.fixer_container.error_level2 | quote }} + {{- if .Values.debezium.fixer_container.startupProbe.enabled }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.debezium.fixer_container.startupProbe "enabled") "context" $) | nindent 10 }} + {{- end }} + {{- if .Values.debezium.fixer_container.livenessProbe.enabled }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.debezium.fixer_container.livenessProbe "enabled") "context" $) | nindent 10 }} + {{- end }} + {{- if .Values.debezium.fixer_container.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.debezium.fixer_container.readinessProbe "enabled") "context" $) | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: debezium-service + labels: + app: debezium-service +spec: + type: ClusterIP + ports: + - name: debezium + protocol: TCP + port: {{ .Values.debezium.port }} + selector: + app: debezium +--- +{{- end }} diff --git a/helm/reporting/templates/kafka-client.yaml b/helm/reporting/templates/kafka-client.yaml new file mode 100644 index 0000000..9ee1088 --- /dev/null +++ b/helm/reporting/templates/kafka-client.yaml @@ -0,0 +1,22 @@ +{{ if .Values.kafkaClient.enabled }} +apiVersion: v1 +kind: Pod +metadata: + name: kafka-client +spec: + containers: + - name: kafka-client + image: {{ .Values.kafkaClient.image }} + command: + - sh + - -c + - "exec tail -f /dev/null" + {{ if .Values.kafkaClient.db_secretref }} + env: + - name: DB_PASS + valueFrom: + secretKeyRef: + name: {{ .Values.kafkaClient.db_secretref.name }} + key: {{ .Values.kafkaClient.db_secretref.key }} + {{ end }} +{{ end }} diff --git a/helm/reporting/values.yaml b/helm/reporting/values.yaml new file mode 100644 index 0000000..178a98c --- /dev/null +++ b/helm/reporting/values.yaml @@ -0,0 +1,197 @@ +debezium: + enabled: true + replicaCount: 1 + image: debezium/connect:1.7.0.Final + imagePullPolicy: Always + port: 8083 + startupProbe: + enabled: true + httpGet: + path: /connectors + port: 8083 + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 30 + successThreshold: 1 + livenessProbe: + enabled: true + httpGet: + path: /connectors + port: 8083 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + readinessProbe: + enabled: true + httpGet: + path: /connectors + port: 8083 + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + fixer_container: + image: mosipqa/debezium-fixer:develop + imagePullPolicy: Always + startup_delay: "30s" + probe_time_period: "30s" + error_level1: "failed" + error_level2: "PSQLException: An I/O error occurred while sending to the backend" + startupProbe: + enabled: true + exec: + command: + - '/bin/bash' + - '-c' + - 'CHK_ENTRYPOINT=$( ps | grep -v "grep" | grep -c ./docker-entrypoint.sh ); + if [ $CHK_ENTRYPOINT = 0 ]; then + echo "ENTRYPOINT Scipt not running; EXITING"; + exit 1; + fi' + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 30 + successThreshold: 1 + livenessProbe: + enabled: true + exec: + command: + - '/bin/bash' + - '-c' + - 'CHK_ENTRYPOINT=$( ps | grep -v "grep" | grep -c ./docker-entrypoint.sh ); + if [ $CHK_ENTRYPOINT = 0 ]; then + echo "ENTRYPOINT Scipt not running; EXITING"; + exit 1; + fi' + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + readinessProbe: + enabled: true + exec: + command: + - '/bin/bash' + - '-c' + - 'CHK_ENTRYPOINT=$( ps | grep -v "grep" | grep -c ./docker-entrypoint.sh ); + if [ $CHK_ENTRYPOINT = 0 ]; then + echo "ENTRYPOINT Scipt not running; EXITING"; + exit 1; + fi' + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + + kafka: + server: "kafka:9092" + groupId: "1" + topics: + offset: debez-connect-offsets + config: debez-connect-configs + status: debez-connect-status + +kafkaClient: + enabled: true + image: confluentinc/cp-kafka + db_secretref: + name: postgres-postgresql + key: postgresql-password + +bitnami-kafka: + fullnameOverride: kafka + enabled: true + replicaCount: 4 + persistence: + enabled: false + zookeeper: + fullnameOverride: kafka-zookeeper + enabled: true + replicaCount: 4 + persistence: + enabled: false + +confluentinc: + enabled: true + + cp-zookeeper: + enabled: false + cp-kafka: + enabled: false + cp-kafka-rest: + enabled: false + cp-schema-registry: + enabled: false + cp-ksql-server: + enabled: false + cp-control-center: + enabled: false + + cp-kafka-connect: + enabled: true + + nameOverride: es-connect + fullnameOverride: es-connect + + replicaCount: 1 + + image: mosipqa/es-kafka-connect + imageTag: develop + imagePullPolicy: Always + + ## Configure extra options for liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + ## + + + kafka: + bootstrapServers: kafka:9092 + + prometheus: + jmx: + enabled: false + + configurationOverrides: + config.storage.replication.factor: "1" + key.converter.schemas.enable: "false" + offset.storage.replication.factor: "1" + plugin.path: /usr/share/java,/usr/share/confluent-hub-components + replication.factor: "1" + status.storage.replication.factor: "1" + value.converter.schemas.enable: "false" + zookeeper.connect: zookeeper:2181 + customEnv: {} + +bitnami-elasticsearch: + enabled: true + global: + kibanaEnabled: true + data: + fullnameOverride: elasticsearch-data + heapSize: 728m + persistence: + enabled: false + resources: + requests: + memory: 512Mi + master: + fullnameOverride: elasticsearch-master + heapSize: 1024m + persistence: + enabled: false + resources: + requests: + memory: 728Mi + coordinating: + fullnameOverride: elasticsearch-coordinating-only + kibana: + fullnameOverride: kibana + persistence: + enabled: false