diff --git a/deploy/data-archive/README.md b/deploy/data-archive/README.md new file mode 100644 index 00000000..cb97246c --- /dev/null +++ b/deploy/data-archive/README.md @@ -0,0 +1,62 @@ +# Database Archiving Configuration + +This configuration file is used for setting up database connections and defining archiving parameters. Please follow the guidelines below to fill in the required information. + +## Database Connections + +### Archive Database Connection (archive_db) + +- `db_name`: Name of the archive database. +- `host`: Destination host for the archive database. +- `port`: Port number for the archive database connection. +- `su_user`: Superuser for the archive database. +- `su_user_pwd`: Password for the superuser. +- `db_pwd`: Password for the archive database. +- `archivehost`: Destination host for the archive database. +- `archiveport`: Port number for the archive database connection. +- `archiveuname`: Archive database username. +- `archive_dbname`: Archive database name. +- `archive_schemaname`: Archive schema name. +- `archive_db_password`: Password for the archive database. + +### Source Database Connections (source_db) + +For each source database (audit, credential, esignet, ida, idrepo, kernel, master, pms, prereg, regprc, resident), provide the following information: + +- `source__host`: Source database host. +- `source__port`: Port number for the source database connection. +- `source__uname`: Source database username. +- `source__dbname`: Source database name. +- `source__schemaname`: Source schema name. +- `source__db_pass`: Password for the source database. + +- `provide_db_names_to_archive`: Comma-separated list of database names to archive (e.g., "AUDIT,CREDENTIAL,IDA,.....").(in CAPS) + + +## Container Volume Path +container_volume_path: Path where JSON files containing information about all databases will be stored + +## Archiving Information (all_db_tables_info) + +For each database, specify tables_info with details for archiving. Example: + +```yaml +audit: + tables_info: + - source_table: "app_audit_log" + archive_table: "mosip_audit_app_audit_log" + id_column: "log_id" + date_column: "log_dtimes" + retention_days: 30 + operation_type: "archive_delete" + +source_table: Name of the table in the source database. +archive_table: Name of the table in the archive database. +id_column: Column representing the unique identifier. +date_column: Column representing the date of the record. +retention_days: Number of days to retain the archived data. +operation_type: Type of operation for archiving (e.g., archive_delete, delete, none). +- Delete: Delete records from the source table. +- Archive and Delete: Archive records to an archive table and then delete them from the source table. +- Archive (No Delete): Archive records to an archive table without deleting them from the source table. +- None: Skip archival for the specified table. diff --git a/deploy/data-archive/delete.sh b/deploy/data-archive/delete.sh new file mode 100755 index 00000000..521fad2e --- /dev/null +++ b/deploy/data-archive/delete.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Uninstalls data-archive +## Usage: ./delete.sh [kubeconfig] + +if [ $# -ge 1 ] ; then + export KUBECONFIG=$1 +fi + +function deleting_data-archive() { + NS=data-archive + while true; do + read -p "Are you sure you want to delete data-archive helm charts?(Y/n) " yn + if [ $yn = "Y" ] + then + helm -n $NS delete data-archive + break + else + break + fi + done + return 0 +} + +# set commands for error handling. +set -e +set -o errexit ## set -e : exit the script if any statement returns a non-true return value +set -o nounset ## set -u : exit the script if you try to use an uninitialised variable +set -o errtrace # trace ERR through 'time command' and other functions +set -o pipefail # trace ERR through pipes +deleting_data-archive # calling function diff --git a/deploy/data-archive/install.sh b/deploy/data-archive/install.sh new file mode 100755 index 00000000..d54e8086 --- /dev/null +++ b/deploy/data-archive/install.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Installs data-archive +## Usage: ./install.sh [kubeconfig] + +if [ $# -ge 1 ]; then + export KUBECONFIG=$1 +fi + +NS=data-archive +CHART_VERSION=0.0.1-develop + +echo Create $NS namespace +kubectl create ns $NS + +function installing_data-archive() { + echo Updating repos + helm repo add mosip https://mosip.github.io/mosip-helm + helm repo update + + read -p "Is values.yaml for data-archive chart set correctly as part of Pre-requisites?(Y/n) " yn; + if [ "$yn" != "Y" ]; then + echo "ERROR: values.yaml not set correctly; EXITING;"; + exit 1; + fi + + read -p "Please enter the time(hr) to run the cronjob every day (time: 0-23) : " time + if [ -z "$time" ]; then + echo "ERROR: Time cannot be empty; EXITING;"; + exit 1; + fi + if ! [ $time -eq $time ] 2>/dev/null; then + echo "ERROR: Time $time is not a number; EXITING;"; + exit 1; + fi + if [ $time -gt 23 ] || [ $time -lt 0 ]; then + echo "ERROR: Time should be in range ( 0-23 ); EXITING;"; + exit 1; + fi + + read -p "Is archival running for sandbox installation? (Y/N): " archival_running + if [ "$archival_running" == "Y" ]; then + echo "Sandbox installation selected. This will use superuser PostgreSQL secrets for creating archivedb." + super_user_password=$(kubectl get secret --namespace postgres postgres-postgresql -o jsonpath={.data.postgres-password} | base64 --decode) + echo "Common secrets will be used as passwords for all the db users." + db_common_password=$(kubectl get secret --namespace postgres db-common-secrets -o jsonpath={.data.db-dbuser-password} | base64 --decode) + set_db_pwd="--set databases.archive_db.su_user_pwd=$super_user_password \ + --set databases.source_db.source_audit_db_pass=$db_common_password \ + --set databases.source_db.source_credential_db_pass=$db_common_password \ + --set databases.source_db.source_esignet_db_pass=$db_common_password \ + --set databases.source_db.source_ida_db_pass=$db_common_password \ + --set databases.source_db.source_idrepo_db_pass=$db_common_password \ + --set databases.source_db.source_kernel_db_pass=$db_common_password \ + --set databases.source_db.source_master_db_pass=$db_common_password \ + --set databases.source_db.source_pms_db_pass=$db_common_password \ + --set databases.source_db.source_prereg_db_pass=$db_common_password \ + --set databases.source_db.source_regprc_db_pass=$db_common_password \ + --set databases.source_db.source_resident_db_pass=$db_common_password \ + --set databases.archive_db.db_pwd=$db_common_password \ + --set databases.archive_db.archive_db_password=$db_common_password" + + elif [ "$archival_running" == "N" ]; then + echo "Other installation selected.This will Use individual secrets for db passwords from values.yaml" + set_db_pwd="" + else + echo "Incorrect input; EXITING;" + exit 1; + fi + + # Install data-archive + helm -n $NS install data-archive mosip/data-archive --set crontime="0 $time * * *" -f values.yaml $set_db_pwd --version $CHART_VERSION + + echo Installed data-archive + return 0 +} + +# set commands for error handling. +set -e +set -o errexit ## set -e : exit the script if any statement returns a non-true return value +set -o nounset ## set -u : exit the script if you try to use an uninitialised variable +set -o errtrace # trace ERR through 'time command' and other functions +set -o pipefail # trace ERR through pipes +installing_data-archive # calling function diff --git a/deploy/data-archive/values.yaml b/deploy/data-archive/values.yaml new file mode 100644 index 00000000..64e3cd4f --- /dev/null +++ b/deploy/data-archive/values.yaml @@ -0,0 +1,281 @@ +databases: + archive_db: + db_name: "mosip_archive" + host: "" + port: 5432 + su_user: "postgres" + su_user_pwd: "" + db_pwd: "" + dml: 0 + archivehost: "" + archiveport: 5432 + archiveuname: "archiveuser" + archive_dbname: "mosip_archive" + archive_schemaname: "archive" + archive_db_password: "" + source_db: + provide_db_names_to_archive: "AUDIT,CREDENTIAL,IDA" + source_audit_host: "" + source_audit_port: 5432 + source_audit_uname: "audituser" + source_audit_dbname: "mosip_audit" + source_audit_schemaname: "audit" + source_audit_db_pass: "" + source_credential_host: "" + source_credential_port: 5432 + source_credential_uname: "credentialuser" + source_credential_dbname: "mosip_credential" + source_credential_schemaname: "credential" + source_credential_db_pass: "" + source_esignet_host: "" + source_esignet_port: 5432 + source_esignet_uname: "esignetuser" + source_esignet_dbname: "mosip_esignet" + source_esignet_schemaname: "esignet" + source_esignet_db_pass: "" + source_ida_host: "" + source_ida_port: 5432 + source_ida_uname: "idauser" + source_ida_dbname: "mosip_ida" + source_ida_schemaname: "ida" + source_ida_db_pass: "" + source_idrepo_host: "" + source_idrepo_port: 5432 + source_idrepo_uname: "idrepouser" + source_idrepo_dbname: "mosip_idrepo" + source_idrepo_schemaname: "idrepo" + source_idrepo_db_pass: "" + source_kernel_host: "" + source_kernel_port: 5432 + source_kernel_uname: "kerneluser" + source_kernel_dbname: "mosip_kernel" + source_kernel_schemaname: "kernel" + source_kernel_db_pass: "" + source_master_host: "" + source_master_port: 5432 + source_master_uname: "masteruser" + source_master_dbname: "mosip_master" + source_master_schemaname: "master" + source_master_db_pass: "" + source_pms_host: "" + source_pms_port: 5432 + source_pms_uname: "pmsuser" + source_pms_dbname: "mosip_pms" + source_pms_schemaname: "pms" + source_pms_db_pass: "" + source_prereg_host: "" + source_prereg_port: 5432 + source_prereg_uname: "prereguser" + source_prereg_dbname: "mosip_prereg" + source_prereg_schemaname: "prereg" + source_prereg_db_pass: "" + source_regprc_host: "" + source_regprc_port: 5432 + source_regprc_uname: "regprcuser" + source_regprc_dbname: "mosip_regprc" + source_regprc_schemaname: "regprc" + source_regprc_db_pass: "" + source_resident_host: "" + source_resident_port: 5432 + source_resident_uname: "residentuser" + source_resident_dbname: "mosip_resident" + source_resident_schemaname: "resident" + source_resident_db_pass: "" + container_volume_path: "/all-db-info-json" + all_db_tables_info: + audit: + tables_info: + - source_table: "app_audit_log" + archive_table: "mosip_audit_app_audit_log" + id_column: "log_id" + date_column: "log_dtimes" + retention_days: 30 + operation_type: "archive_delete" + credential: + tables_info: + - source_table: "credential_transaction" + archive_table: "mosip_credential_credential_transaction" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: "archive_delete" + esignet: + tables_info: + - source_table: "consent_history" + archive_table: "mosip_esignet_consent_history" + id_column: "id" + date_column: "cr_dtimes" + retention_days: 30 + operation_type: "none" + ida: + tables_info: + - source_table: "credential_event_store" + archive_table: "mosip_ida_credential_event_store" + id_column: "event_id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: "archive_delete" + - source_table: "otp_transaction" + archive_table: "mosip_ida_otp_transaction" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: 'delete' + idrepo: + tables_info: + - source_table: "anonymous_profile" + archive_table: "mosip_idrepo_anonymous_profile" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: "archive_delete" + - source_table: "credential_request_status" + archive_table: "mosip_idrepo_credential_request_status" + id_column: "individual_id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: "archive_delete" + - source_table: "uin_draft" + archive_table: "mosip_idrepo_uin_draft" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: "archive_delete" + kernel: + tables_info: + - source_table: "otp_transaction" + archive_table: "mosip_kernel_otp_transaction" + id_column: "id" + date_column: "generated_dtimes" + retension_days: 7 + operation_type: "delete" + master: + tables_info: + - source_table: "bulkupload_transaction" + archive_table: "mosip_master_bulkupload_transaction" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 91 + operation_type: "archive_delete" + - source_table: "device_master_h" + archive_table: "mosip_master_device_master_h" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 365 + operation_type: "archive_delete" + - source_table: "machine_master_h" + archive_table: "mosip_master_machine_master_h" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "registration_center_h" + archive_table: "mosip_master_registration_center_h" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 365 + operation_type: "archive_delete" + - source_table: "user_detail_h" + archive_table: "mosip_master_user_detail_h" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "zone_user_h" + archive_table: "mosip_master_zone_user_h" + id_column: "usr_id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + pms: + tables_info: + - source_table: "auth_policy_h" + archive_table: "mosip_pms_auth_policy_h" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "secure_biometric_interface_h" + archive_table: "mosip_pms_secure_biometric_interface_h" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "partner_h" + archive_table: "mosip_pms_partner_h" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + prereg: + tables_info: + - source_table: "otp_transaction" + archive_table: "mosip_prereg_otp_transaction" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: "delete" + regprc: + tables_info: + - source_table: "abis_response_det" + archive_table: "mosip_regprc_abis_response_det" + id_column: "abis_resp_id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "abis_response" + archive_table: "mosip_regprc_abis_response" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "abis_request" + archive_table: "mosip_regprc_abis_request" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "reg_demo_dedupe_list" + archive_table: "mosip_regprc_reg_demo_dedupe_list" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "registration_transaction" + archive_table: "mosip_regprc_registration_transaction" + id_column: "regtrn_id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + resident: + tables_info: + - source_table: "otp_transaction" + archive_table: "mosip_resident_otp_transaction" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: "delete" + - source_table: "resident_grievance_ticket" + archive_table: "mosip_resident_grievance_ticket" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 365 + operation_type: "archive_delete" + - source_table: "resident_session" + archive_table: "mosip_resident_session" + id_column: "session_id" + date_column: "login_dtimes" + retension_days: 30 + operation_type: "archive_delete" + - source_table: "resident_transaction" + archive_table: "mosip_resident_transaction" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 365 + operation_type: "archive_delete" + - source_table: "resident_user_actions" + archive_table: "mosip_resident_user_actions" + id_column: "ida_token" + date_column: "last_bell_notif_click_dtimes" + retension_days: 365 + operation_type: "archive_delete" \ No newline at end of file diff --git a/helm/data-archive/.gitignore b/helm/data-archive/.gitignore new file mode 100644 index 00000000..ee3892e8 --- /dev/null +++ b/helm/data-archive/.gitignore @@ -0,0 +1 @@ +charts/ diff --git a/helm/data-archive/.helmignore b/helm/data-archive/.helmignore new file mode 100644 index 00000000..35e62f99 --- /dev/null +++ b/helm/data-archive/.helmignore @@ -0,0 +1,24 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +.chart.lock diff --git a/helm/data-archive/Chart.yaml b/helm/data-archive/Chart.yaml new file mode 100644 index 00000000..2a706d0e --- /dev/null +++ b/helm/data-archive/Chart.yaml @@ -0,0 +1,19 @@ +apiVersion: v2 +name: data-archive +description: A Helm chart for Kubernetes to initialize mosip_archive Postgres DB and execute py script to archive data from source db to archive db +type: application +version: 0.0.1-develop +dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +home: https://mosip.io +keywords: + - postgres + - db + - database +maintainers: + - email: info@mosip.io + name: MOSIP diff --git a/helm/data-archive/templates/_helpers.tpl b/helm/data-archive/templates/_helpers.tpl new file mode 100644 index 00000000..6b92a3a0 --- /dev/null +++ b/helm/data-archive/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* +Return the proper image name +*/}} +{{- define "data-archive.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global) }} +{{- end -}} + +{{/* +Return the proper image name (for the init container volume-permissions image) +*/}} +{{- define "data-archive.volumePermissions.image" -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.volumePermissions.image "global" .Values.global ) -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names +*/}} +{{- define "data-archive.imagePullSecrets" -}} +{{- include "common.images.pullSecrets" (dict "images" (list .Values.image .Values.volumePermissions.image) "global" .Values.global) -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "data-archive.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (printf "%s-foo" (include "common.names.fullname" .)) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Compile all warnings into a single message. +*/}} +{{- define "data-archive.validateValues" -}} +{{- $messages := list -}} +{{- $messages := append $messages (include "data-archive.validateValues.foo" .) -}} +{{- $messages := append $messages (include "data-archive.validateValues.bar" .) -}} +{{- $messages := without $messages "" -}} +{{- $message := join "\n" $messages -}} + +{{- if $message -}} +{{- printf "\nVALUES VALIDATION:\n%s" $message -}} +{{- end -}} +{{- end -}} + +{{/* +Return podAnnotations +*/}} +{{- define "data-archive.podAnnotations" -}} +{{- if .Values.podAnnotations }} +{{ include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) }} +{{- end }} +{{- if and .Values.metrics.enabled .Values.metrics.podAnnotations }} +{{ include "common.tplvalues.render" (dict "value" .Values.metrics.podAnnotations "context" $) }} +{{- end }} +{{- end -}} + +{{/* Create the name for restart cronjob */}} +{{- define "data-archive.cronjob" -}} +{{ default (printf "cronjob-%s" (include "common.names.fullname" .)) .Values.serviceAccount.name }} +{{- end -}} \ No newline at end of file diff --git a/helm/data-archive/templates/all-db-tables-info-configmap.yaml b/helm/data-archive/templates/all-db-tables-info-configmap.yaml new file mode 100644 index 00000000..e34b2cc6 --- /dev/null +++ b/helm/data-archive/templates/all-db-tables-info-configmap.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: your-component-name + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + audit_archive_table_info: {{ .Values.databases.all_db_tables_info.audit | toJson | quote }} + credential_archive_table_info: {{ .Values.databases.all_db_tables_info.credential | toJson | quote }} + esignet_archive_table_info: {{ .Values.databases.all_db_tables_info.esignet | toJson | quote }} + ida_archive_table_info: {{ .Values.databases.all_db_tables_info.ida | toJson | quote }} + idrepo_archive_table_info: {{ .Values.databases.all_db_tables_info.idrepo | toJson | quote }} + kernel_archive_table_info: {{ .Values.databases.all_db_tables_info.kernel | toJson | quote }} + master_archive_table_info: {{ .Values.databases.all_db_tables_info.master | toJson | quote }} + pms_archive_table_info: {{ .Values.databases.all_db_tables_info.pms | toJson | quote }} + prereg_archive_table_info: {{ .Values.databases.all_db_tables_info.prereg | toJson | quote }} + regprc_archive_table_info: {{ .Values.databases.all_db_tables_info.regprc | toJson | quote }} + resident_archive_table_info: {{ .Values.databases.all_db_tables_info.resident | toJson | quote }} \ No newline at end of file diff --git a/helm/data-archive/templates/archive-db-secret.yaml b/helm/data-archive/templates/archive-db-secret.yaml new file mode 100644 index 00000000..ee4c424e --- /dev/null +++ b/helm/data-archive/templates/archive-db-secret.yaml @@ -0,0 +1,19 @@ +# secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: archive-db-secrets + namespace: {{ .Release.Namespace }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: postgres + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +type: Opaque +data: + SU_USER_PWD: {{ .Values.databases.archive_db.su_user_pwd | b64enc | quote }} + DBUSER_PWD: {{ .Values.databases.archive_db.db_pwd | b64enc | quote }} + ARCHIVE_DB_PASS: {{ .Values.databases.archive_db.archive_db_password | b64enc | quote }} diff --git a/helm/data-archive/templates/archive-env-configmap.yaml b/helm/data-archive/templates/archive-env-configmap.yaml new file mode 100644 index 00000000..7833668a --- /dev/null +++ b/helm/data-archive/templates/archive-env-configmap.yaml @@ -0,0 +1,81 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: db-archive-init-env-config + namespace: {{ .Release.Namespace }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: postgres + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + MOSIP_DB_NAME: {{ .Values.databases.archive_db.db_name | quote }} + SU_USER: {{ .Values.databases.archive_db.su_user | quote }} + DB_SERVERIP: {{ .Values.databases.archive_db.host | quote }} + DB_PORT: {{ .Values.databases.archive_db.port | quote }} + DML_FLAG: {{ .Values.databases.archive_db.dml | quote }} + DB_NAMES: {{ .Values.databases.source_db.provide_db_names_to_archive | quote }} + AUDIT_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_audit_host | quote }} + AUDIT_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_audit_port | quote }} + AUDIT_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_audit_uname | quote }} + AUDIT_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_audit_dbname | quote }} + AUDIT_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_audit_schemaname | quote }} + CREDENTIAL_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_credential_host | quote }} + CREDENTIAL_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_credential_port | quote }} + CREDENTIAL_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_credential_uname | quote }} + CREDENTIAL_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_credential_dbname | quote }} + CREDENTIAL_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_credential_schemaname | quote }} + ESIGNET_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_esignet_host | quote }} + ESIGNET_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_esignet_port | quote }} + ESIGNET_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_esignet_uname | quote }} + ESIGNET_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_esignet_dbname | quote }} + ESIGNET_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_esignet_schemaname | quote }} + IDA_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_ida_host | quote }} + IDA_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_ida_port | quote }} + IDA_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_ida_uname | quote }} + IDA_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_ida_dbname | quote }} + IDA_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_ida_schemaname | quote }} + IDREPO_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_idrepo_host | quote }} + IDREPO_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_idrepo_port | quote }} + IDREPO_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_idrepo_uname | quote }} + IDREPO_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_idrepo_dbname | quote }} + IDREPO_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_idrepo_schemaname | quote }} + KERNEL_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_kernel_host | quote }} + KERNEL_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_kernel_port | quote }} + KERNEL_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_kernel_uname | quote }} + KERNEL_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_kernel_dbname | quote }} + KERNEL_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_kernel_schemaname | quote }} + MASTER_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_master_host | quote }} + MASTER_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_master_port | quote }} + MASTER_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_master_uname | quote }} + MASTER_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_master_dbname | quote }} + MASTER_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_master_schemaname | quote }} + PMS_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_pms_host | quote }} + PMS_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_pms_port | quote }} + PMS_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_pms_uname | quote }} + PMS_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_pms_dbname | quote }} + PMS_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_pms_schemaname | quote }} + PREREG_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_prereg_host | quote }} + PREREG_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_prereg_port | quote }} + PREREG_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_prereg_uname | quote }} + PREREG_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_prereg_dbname | quote }} + PREREG_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_prereg_schemaname | quote }} + REGPRC_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_regprc_host | quote }} + REGPRC_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_regprc_port | quote }} + REGPRC_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_regprc_uname | quote }} + REGPRC_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_regprc_dbname | quote }} + REGPRC_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_regprc_schemaname | quote }} + RESIDENT_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_resident_host | quote }} + RESIDENT_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_resident_port | quote }} + RESIDENT_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_resident_uname | quote }} + RESIDENT_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_resident_dbname | quote }} + RESIDENT_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_resident_schemaname | quote }} + ARCHIVE_DB_HOST: {{ .Values.databases.archive_db.archivehost | quote }} + ARCHIVE_DB_PORT: {{ .Values.databases.archive_db.archiveport | quote }} + ARCHIVE_DB_UNAME: {{ .Values.databases.archive_db.archiveuname | quote }} + ARCHIVE_DB_NAME: {{ .Values.databases.archive_db.archive_dbname | quote }} + ARCHIVE_SCHEMA_NAME: {{ .Values.databases.archive_db.archive_schemaname | quote }} + CONTAINER_VOLUME_PATH: {{ .Values.databases.container_volume_path | quote }} \ No newline at end of file diff --git a/helm/data-archive/templates/cronjob.yaml b/helm/data-archive/templates/cronjob.yaml new file mode 100644 index 00000000..5a808e78 --- /dev/null +++ b/helm/data-archive/templates/cronjob.yaml @@ -0,0 +1,42 @@ +apiVersion: {{ include "common.capabilities.cronjob.apiVersion" $ }} +kind: CronJob +metadata: + name: {{ template "data-archive.cronjob" $ }} + namespace: {{ $.Release.Namespace }} +spec: + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 # remove jobs which are successfully executed + failedJobsHistoryLimit: 1 # except 1 recent failed job, remove jobs which are not successfully executed + #schedule: '*/3 * * * *' # cron spec of time, here, 8 o'clock + schedule: {{ $.Values.crontime }} + jobTemplate: + spec: + backoffLimit: 2 # this has very low chance of failing, as all this does + # is prompt kubernetes to schedule new replica set for + # the deployment + # activeDeadlineSeconds: 600 # timeout, makes most sense with + # "waiting for rollout" variant specified below + template: + spec: + restartPolicy: Never + containers: + - name: {{ template "data-archive.serviceAccountName" $ }} + image: {{ template "data-archive.image" $ }} + imagePullPolicy: {{ $.Values.image.pullPolicy }} + volumeMounts: + - name: config-volume + mountPath: /all-db-info-json + env: + - name: container_user + value: {{ $.Values.containerSecurityContext.runAsUser }} + envFrom: + - configMapRef: + name: db-archive-init-env-config + - secretRef: + name: source-db-secrets + - secretRef: + name: archive-db-secrets + volumes: + - name: config-volume + configMap: + name: {{ .Release.Name }} \ No newline at end of file diff --git a/helm/data-archive/templates/serviceaccount.yaml b/helm/data-archive/templates/serviceaccount.yaml new file mode 100644 index 00000000..cedd6dc6 --- /dev/null +++ b/helm/data-archive/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + name: {{ template "data-archive.serviceAccountName" . }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + namespace: {{ .Release.Namespace }} \ No newline at end of file diff --git a/helm/data-archive/templates/source-db-secret.yaml b/helm/data-archive/templates/source-db-secret.yaml new file mode 100644 index 00000000..2ed02932 --- /dev/null +++ b/helm/data-archive/templates/source-db-secret.yaml @@ -0,0 +1,27 @@ +# secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: source-db-secrets + namespace: {{ .Release.Namespace }} + labels: {{- include "common.labels.standard" . | nindent 4 }} + app.kubernetes.io/component: postgres + {{- if .Values.commonLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +type: Opaque +data: + AUDIT_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_audit_db_pass | b64enc | quote }} + CREDENTIAL_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_credential_db_pass | b64enc | quote }} + ESIGNET_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_esignet_db_pass | b64enc | quote }} + IDA_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_ida_db_pass | b64enc | quote }} + IDREPO_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_idrepo_db_pass | b64enc | quote }} + KERNEL_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_kernel_db_pass | b64enc | quote }} + MASTER_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_master_db_pass | b64enc | quote }} + PMS_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_pms_db_pass | b64enc | quote }} + PREREG_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_prereg_db_pass | b64enc | quote }} + REGPRC_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_regprc_db_pass | b64enc | quote }} + RESIDENT_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_resident_db_pass | b64enc | quote }} \ No newline at end of file diff --git a/helm/data-archive/values.yaml b/helm/data-archive/values.yaml new file mode 100644 index 00000000..c99093c7 --- /dev/null +++ b/helm/data-archive/values.yaml @@ -0,0 +1,589 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +# global: +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName +# storageClass: myStorageClass + +## Add labels to all the deployed resources +## +commonLabels: + app.kubernetes.io/component: mosip + +## Add annotations to all the deployed resources +## +commonAnnotations: {} + +## Kubernetes Cluster Domain +## +clusterDomain: cluster.local + +## Extra objects to deploy (value evaluated as a template) +## +extraDeploy: [] + +## Number of nodes +## +replicaCount: 1 + +image: + registry: docker.io + repository: mosipqa/data-archive + tag: develop + + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + +## +# existingConfigmap: + +## Command and args for running the container (set to default if not set). Use array form +## +command: [] +args: [] + +## Deployment pod host aliases +## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ +## +hostAliases: [] + +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: + cpu: 1000m + memory: 3500Mi + requests: + cpu: 1000m + memory: 3500Mi + +# additionalResources: + ## Specify any JAVA_OPTS string here. These typically will be specified in conjunction with above resources + ## Example: java_opts: "-Xms500M -Xmx500M" + # javaOpts: "-Xms2600M -Xmx2600M" + +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +## Clamav container already runs as 'mosip' user, so we may not need to enable this +containerSecurityContext: + enabled: false + runAsUser: mosip + runAsNonRoot: true + +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +## +podSecurityContext: + enabled: false + fsGroup: 1001 + +## Pod affinity preset +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity +## Allowed values: soft, hard +## +podAffinityPreset: "" + +## Pod anti-affinity preset +## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity +## Allowed values: soft, hard +## +podAntiAffinityPreset: soft + +## Node affinity preset +## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity +## Allowed values: soft, hard +## +nodeAffinityPreset: + ## Node affinity type + ## Allowed values: soft, hard + ## + type: "" + ## Node label key to match + ## E.g. + ## key: "kubernetes.io/e2e-az-name" + ## + key: "" + ## Node label values to match + ## E.g. + ## values: + ## - e2e-az1 + ## - e2e-az2 + ## + values: [] + +## Affinity for pod assignment. Evaluated as a template. +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Node labels for pod assignment. Evaluated as a template. +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +## Some common set of users are created across all postgres servers with same password. TODO: change this later. +## These user names are currently hardcoded in the sql scripts, so can't change from here. + +nodeSelector: {} + +## Tolerations for pod assignment. Evaluated as a template. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Pod extra labels +## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +## +podLabels: {} + +## Annotations for server pods. +## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +## +podAnnotations: {} + +## pods' priority. +## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +## +# priorityClassName: "" + +## lifecycleHooks for the container to automate configuration before or after startup. +## +lifecycleHooks: {} + +## Custom Liveness probes for +## +customLivenessProbe: {} + +## Custom Rediness probes +## +customReadinessProbe: {} + +## Update strategy - only really applicable for deployments with RWO PVs attached +## If replicas = 1, an update can get "stuck", as the previous pod remains attached to the +## PV, and the "incoming" pod can never start. Changing the strategy to "Recreate" will +## terminate the single previous pod, so that the new, incoming pod can attach to the PV +## +updateStrategy: + type: RollingUpdate + +## Additional environment variables to set +## Example: +## extraEnvVars: +## - name: FOO +## value: "bar" +## +extraEnvVars: [] + +## ConfigMap with extra environment variables +## +# extraEnvVarsCM: + # - s3 + +## Secret with extra environment variables +## +# extraEnvVarsSecret: + # - s3 + +## Extra volumes to add to the deployment +## +extraVolumes: [] + +## Extra volume mounts to add to the container +## +extraVolumeMounts: [] + +## Add init containers to the pods. +## Example: +## initContainers: +## - name: your-image-name +## image: your-image +## imagePullPolicy: Always +## ports: +## - name: portname +## containerPort: 1234 +## +## initContainers: {} + +## Add sidecars to the pods. +## Example: +## sidecars: +## - name: your-image-name +## image: your-image +## imagePullPolicy: Always +## ports: +## - name: portname +## containerPort: 1234 +## +sidecars: {} + +persistence: + enabled: false + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack). + ## + # storageClass: "-" + ## + ## If you want to reuse an existing claim, you can pass the name of the PVC using + ## the existingClaim variable + # existingClaim: your-claim + ## ReadWriteMany not supported by AWS gp2 + storageClass: + accessModes: + - ReadWriteOnce + size: 10M + existingClaim: + # Dir where config and keys are written inside container + mountDir: + +## Init containers parameters: +## volumePermissions: Change the owner and group of the persistent volume mountpoint to runAsUser:fsGroup values from the securityContext section. +## +volumePermissions: + enabled: false + image: + registry: docker.io + repository: bitnami/bitnami-shell + tag: "10" + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + pullSecrets: [] + ## - myRegistryKeySecretName + ## Init containers' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## + # limits: {} + ## cpu: 100m + ## memory: 128Mi + ## + # requests: {} + ## cpu: 100m + ## memory: 128Mi + ## + +## Specifies whether RBAC resources should be created +## +rbac: + create: true + +## Specifies whether a ServiceAccount should be created +## +serviceAccount: + create: true + ## The name of the ServiceAccount to use. + ## If not set and create is true, a name is generated using the fullname template + ## + name: + +## Admin swagger should have only internal access. Hence linked to internal gateway +istio: + enabled: false + gateways: + - istio-system/internal + +crontime: "0 4 * * *" ## run cronjob every day at 4 AM in GMT which is 9.30 AM in IST (time hr: 0-23 ) + +databases: + archive_db: + db_name: "mosip_archive" + host: "api-internal.sandbox.xyz.net" + port: 5432 + su_user: "postgres" + su_user_pwd: "" + db_pwd: "" + dml: 0 + archivehost: "api-internal.sandbox.xyz.net" + archiveport: 5432 + archiveuname: "archiveuser" + archive_dbname: "mosip_archive" + archive_schemaname: "archive" + archive_db_password: "" + source_db: + provide_db_names_to_archive: "AUDIT,IDA" + source_audit_host: "api-internal.sandbox.xyz.net" + source_audit_port: 5432 + source_audit_uname: "audituser" + source_audit_dbname: "mosip_audit" + source_audit_schemaname: "audit" + source_audit_db_pass: "" + source_credential_host: "api-internal.sandbox.xyz.net" + source_credential_port: 5432 + source_credential_uname: "credentialuser" + source_credential_dbname: "mosip_credential" + source_credential_schemaname: "credential" + source_credential_db_pass: "" + source_esignet_host: "api-internal.sandbox.xyz.net" + source_esignet_port: 5432 + source_esignet_uname: "esignetuser" + source_esignet_dbname: "mosip_esignet" + source_esignet_schemaname: "esignet" + source_esignet_db_pass: "" + source_ida_host: "api-internal.sandbox.xyz.net" + source_ida_port: 5432 + source_ida_uname: "idauser" + source_ida_dbname: "mosip_ida" + source_ida_schemaname: "ida" + source_ida_db_pass: "" + source_idrepo_host: "api-internal.sandbox.xyz.net" + source_idrepo_port: 5432 + source_idrepo_uname: "idrepouser" + source_idrepo_dbname: "mosip_idrepo" + source_idrepo_schemaname: "idrepo" + source_idrepo_db_pass: "" + source_kernel_host: "api-internal.sandbox.xyz.net" + source_kernel_port: 5432 + source_kernel_uname: "kerneluser" + source_kernel_dbname: "mosip_kernel" + source_kernel_schemaname: "kernel" + source_kernel_db_pass: "" + source_master_host: "api-internal.sandbox.xyz.net" + source_master_port: 5432 + source_master_uname: "masteruser" + source_master_dbname: "mosip_master" + source_master_schemaname: "master" + source_master_db_pass: "" + source_pms_host: "api-internal.sandbox.xyz.net" + source_pms_port: 5432 + source_pms_uname: "pmsuser" + source_pms_dbname: "mosip_pms" + source_pms_schemaname: "pms" + source_pms_db_pass: "" + source_prereg_host: "api-internal.sandbox.xyz.net" + source_prereg_port: 5432 + source_prereg_uname: "prereguser" + source_prereg_dbname: "mosip_prereg" + source_prereg_schemaname: "prereg" + source_prereg_db_pass: "" + source_regprc_host: "api-internal.sandbox.xyz.net" + source_regprc_port: 5432 + source_regprc_uname: "regprcuser" + source_regprc_dbname: "mosip_regprc" + source_regprc_schemaname: "regprc" + source_regprc_db_pass: "" + source_resident_host: "api-internal.sandbox.xyz.net" + source_resident_port: 5432 + source_resident_uname: "residentuser" + source_resident_dbname: "mosip_resident" + source_resident_schemaname: "resident" + source_resident_db_pass: "" + container_volume_path: "/all-db-info-json" ## volume mount path for table info inside cronjob container + all_db_tables_info: + audit: + tables_info: + - source_table: "app_audit_log" + archive_table: "mosip_audit_app_audit_log" + id_column: "log_id" + date_column: "log_dtimes" + retention_days: 30 + operation_type: "archive_delete" + credential: + tables_info: + - source_table: "credential_transaction" + archive_table: "mosip_credential_credential_transaction" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: "archive_delete" + esignet: + tables_info: + - source_table: "consent_history" + archive_table: "mosip_esignet_consent_history" + id_column: "id" + date_column: "cr_dtimes" + retention_days: 30 + operation_type: "none" + ida: + tables_info: + - source_table: "credential_event_store" + archive_table: "mosip_ida_credential_event_store" + id_column: "event_id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: "archive_delete" + - source_table: "otp_transaction" + archive_table: "mosip_ida_otp_transaction" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: 'delete' + idrepo: + tables_info: + - source_table: "anonymous_profile" + archive_table: "mosip_idrepo_anonymous_profile" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: "archive_delete" + - source_table: "credential_request_status" + archive_table: "mosip_idrepo_credential_request_status" + id_column: "individual_id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: "archive_delete" + - source_table: "uin_draft" + archive_table: "mosip_idrepo_uin_draft" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: "archive_delete" + kernel: + tables_info: + - source_table: "otp_transaction" + archive_table: "mosip_kernel_otp_transaction" + id_column: "id" + date_column: "generated_dtimes" + retension_days: 7 + operation_type: "delete" + master: + tables_info: + - source_table: "bulkupload_transaction" + archive_table: "mosip_master_bulkupload_transaction" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 91 + operation_type: "archive_delete" + - source_table: "device_master_h" + archive_table: "mosip_master_device_master_h" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 365 + operation_type: "archive_delete" + - source_table: "machine_master_h" + archive_table: "mosip_master_machine_master_h" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "registration_center_h" + archive_table: "mosip_master_registration_center_h" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 365 + operation_type: "archive_delete" + - source_table: "user_detail_h" + archive_table: "mosip_master_user_detail_h" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "zone_user_h" + archive_table: "mosip_master_zone_user_h" + id_column: "usr_id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + pms: + tables_info: + - source_table: "auth_policy_h" + archive_table: "mosip_pms_auth_policy_h" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "secure_biometric_interface_h" + archive_table: "mosip_pms_secure_biometric_interface_h" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "partner_h" + archive_table: "mosip_pms_partner_h" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + prereg: + tables_info: + - source_table: "otp_transaction" + archive_table: "mosip_prereg_otp_transaction" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: "delete" + regprc: + tables_info: + - source_table: "abis_response_det" + archive_table: "mosip_regprc_abis_response_det" + id_column: "abis_resp_id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "abis_response" + archive_table: "mosip_regprc_abis_response" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "abis_request" + archive_table: "mosip_regprc_abis_request" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "reg_demo_dedupe_list" + archive_table: "mosip_regprc_reg_demo_dedupe_list" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + - source_table: "registration_transaction" + archive_table: "mosip_regprc_registration_transaction" + id_column: "regtrn_id" + date_column: "cr_dtimes" + retension_days: 183 + operation_type: "archive_delete" + resident: + tables_info: + - source_table: "otp_transaction" + archive_table: "mosip_resident_otp_transaction" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 30 + operation_type: "delete" + - source_table: "resident_grievance_ticket" + archive_table: "mosip_resident_grievance_ticket" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 365 + operation_type: "archive_delete" + - source_table: "resident_session" + archive_table: "mosip_resident_session" + id_column: "session_id" + date_column: "login_dtimes" + retension_days: 30 + operation_type: "archive_delete" + - source_table: "resident_transaction" + archive_table: "mosip_resident_transaction" + id_column: "id" + date_column: "cr_dtimes" + retension_days: 365 + operation_type: "archive_delete" + - source_table: "resident_user_actions" + archive_table: "mosip_resident_user_actions" + id_column: "ida_token" + date_column: "last_bell_notif_click_dtimes" + retension_days: 365 + operation_type: "archive_delete"