diff --git a/README.md b/README.md index 855991b..26833a4 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,7 @@ Security scanning is graciously provided by Prowler. Proowler is the leading ful |------|--------|---------| | [aurora](#module\_aurora) | terraform-aws-modules/rds-aurora/aws | 8.3.0 | | [aurora\_secondary](#module\_aurora\_secondary) | terraform-aws-modules/rds-aurora/aws | 8.3.0 | +| [backup\_restore](#module\_backup\_restore) | ./modules/db-backup-restore | n/a | ## Resources @@ -125,10 +126,17 @@ Security scanning is graciously provided by Prowler. Proowler is the leading ful | [autoscaling\_scale\_out\_cooldown](#input\_autoscaling\_scale\_out\_cooldown) | Cooldown in seconds before allowing further scaling operations after a scale out | `number` | `300` | no | | [autoscaling\_target\_connections](#input\_autoscaling\_target\_connections) | No of connections on which aurora has to scale if predefined\_metric\_type is RDSReaderAverageDatabaseConnections | `number` | `50` | no | | [backup\_retention\_period](#input\_backup\_retention\_period) | The number of days to retain backups for | `number` | `null` | no | +| [bucket\_provider\_type](#input\_bucket\_provider\_type) | Choose what type of provider you want (s3, gcs) | `string` | `"s3"` | no | +| [cluster\_name](#input\_cluster\_name) | Specifies the name of the EKS cluster to deploy the MySQL application on. | `string` | `""` | no | | [create\_monitoring\_role](#input\_create\_monitoring\_role) | Set it to true to create IAM role for Enhanced monitoring. | `bool` | `false` | no | +| [create\_namespace](#input\_create\_namespace) | Specify whether or not to create the namespace if it does not already exist. Set it to true to create the namespace. | `string` | `false` | no | | [create\_random\_password](#input\_create\_random\_password) | Whether to create a random password for the primary database cluster | `bool` | `true` | no | | [create\_security\_group](#input\_create\_security\_group) | Whether to create a security group or not | `bool` | `true` | no | | [database\_name](#input\_database\_name) | The name for an automatically created database on cluster creation | `string` | `""` | no | +| [db\_backup\_config](#input\_db\_backup\_config) | configuration options for MySQL database backups. It includes properties such as the S3 bucket URI, the S3 bucket region, and the cron expression for full backups. | `map(string)` |
{
"bucket_uri": "",
"cron_for_full_backup": "",
"mysql_database_name": ""
}
| no | +| [db\_backup\_enabled](#input\_db\_backup\_enabled) | Specifies whether to enable backups for MySQL database. | `bool` | `false` | no | +| [db\_restore\_config](#input\_db\_restore\_config) | Configuration options for restoring dump to the MySQL database. | `any` |
{
"bucket_uri": "",
"file_name": ""
}
| no | +| [db\_restore\_enabled](#input\_db\_restore\_enabled) | Specifies whether to enable restoring dump to the MySQL database. | `bool` | `false` | no | | [deletion\_protection](#input\_deletion\_protection) | Whether accidental deletion protection is enabled | `bool` | `true` | no | | [enable\_egress](#input\_enable\_egress) | Set it true if allow outbound traffic in rds security group | `bool` | `true` | no | | [enable\_http\_endpoint](#input\_enable\_http\_endpoint) | Whether or not to enable the Data API for a serverless Aurora database engine | `bool` | `false` | no | @@ -148,8 +156,11 @@ Security scanning is graciously provided by Prowler. Proowler is the leading ful | [kms\_key\_arn](#input\_kms\_key\_arn) | The ARN for the KMS encryption key. If creating an encrypted replica, set this to the destination KMS ARN. If storage\_encrypted is set to true and kms\_key\_id is not specified the default KMS key created in your account will be used | `string` | `null` | no | | [long\_query\_time](#input\_long\_query\_time) | To prevent fast-running queries from being logged in the slow query log, specify a value for the shortest query runtime to be logged, in seconds | `number` | `10` | no | | [manage\_master\_user\_password](#input\_manage\_master\_user\_password) | Set to true to allow RDS to manage the master user password in Secrets Manager. Cannot be set if `master_password` is provided | `bool` | `false` | no | +| [master\_password](#input\_master\_password) | The password for the primary cluster | `string` | `null` | no | | [master\_username](#input\_master\_username) | The username for the primary cluster | `string` | `"root"` | no | | [monitoring\_interval](#input\_monitoring\_interval) | The interval, in seconds, between points when Enhanced Monitoring metrics are collected for instances. Set to 0 to disble. Default is 0 | `number` | `0` | no | +| [name](#input\_name) | The name of the RDS instance | `string` | `""` | no | +| [namespace](#input\_namespace) | Name of the Kubernetes namespace where the MYSQL deployment will be deployed. | `string` | `"db"` | no | | [performance\_insights\_enabled](#input\_performance\_insights\_enabled) | Specifies whether Performance Insights is enabled or not | `bool` | `null` | no | | [performance\_insights\_kms\_key\_id](#input\_performance\_insights\_kms\_key\_id) | ARN of KMS key to encrypt performance insights data. | `string` | `null` | no | | [performance\_insights\_retention\_period](#input\_performance\_insights\_retention\_period) | Retention period for performance insights data, Either 7 (7 days) or 731 (2 years). | `number` | `null` | no | @@ -182,6 +193,7 @@ Security scanning is graciously provided by Prowler. Proowler is the leading ful | Name | Description | |------|-------------| +| [rds\_cluster\_database\_name](#output\_rds\_cluster\_database\_name) | Name for an automatically created database on cluster creation | | [rds\_cluster\_endpoint](#output\_rds\_cluster\_endpoint) | The endpoint URL of the Aurora cluster | | [rds\_cluster\_master\_password](#output\_rds\_cluster\_master\_password) | The master password for the Aurora cluster | | [rds\_cluster\_master\_username](#output\_rds\_cluster\_master\_username) | The master username for the Aurora cluster | diff --git a/examples/aurora-global/README.md b/examples/aurora-global/README.md index d767c4d..111ef74 100644 --- a/examples/aurora-global/README.md +++ b/examples/aurora-global/README.md @@ -17,7 +17,7 @@ | Name | Source | Version | |------|--------|---------| -| [aurora](#module\_aurora) | squareops/rds-aurora/aws | n/a | +| [aurora](#module\_aurora) | squareops/rds-aurora/aws | 2.2.1 | | [kms](#module\_kms) | terraform-aws-modules/kms/aws | n/a | | [secondary\_vpc](#module\_secondary\_vpc) | squareops/vpc/aws | n/a | | [vpc](#module\_vpc) | squareops/vpc/aws | n/a | diff --git a/examples/aurora/README.md b/examples/aurora/README.md index 3772d22..8c379c0 100644 --- a/examples/aurora/README.md +++ b/examples/aurora/README.md @@ -16,7 +16,7 @@ | Name | Source | Version | |------|--------|---------| -| [aurora](#module\_aurora) | squareops/rds-aurora/aws | n/a | +| [aurora](#module\_aurora) | squareops/rds-aurora/aws | 2.2.1 | | [kms](#module\_kms) | terraform-aws-modules/kms/aws | n/a | | [vpc](#module\_vpc) | squareops/vpc/aws | n/a | @@ -25,6 +25,8 @@ | Name | Type | |------|------| | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | +| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | ## Inputs @@ -35,6 +37,7 @@ No inputs. | Name | Description | |------|-------------| +| [aurora\_cluster\_database\_name](#output\_aurora\_cluster\_database\_name) | The reader endpoint URL of the Aurora cluster | | [aurora\_cluster\_endpoint](#output\_aurora\_cluster\_endpoint) | The endpoint URL of the Aurora cluster | | [aurora\_cluster\_master\_password](#output\_aurora\_cluster\_master\_password) | The master password for the Aurora cluster | | [aurora\_cluster\_master\_username](#output\_aurora\_cluster\_master\_username) | The master username for the Aurora cluster | diff --git a/examples/aurora/helm/values.yaml b/examples/aurora/helm/values.yaml new file mode 100644 index 0000000..bfd3de9 --- /dev/null +++ b/examples/aurora/helm/values.yaml @@ -0,0 +1,48 @@ +primary: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Addons-Services" + operator: In + values: + - "true" + +secondary: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Addons-Services" + operator: In + values: + - "true" + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Addons-Services" + operator: In + values: + - "true" +backupjob: + resources: + requests: + memory: 100Mi + cpu: 50m + limits: + memory: 200Mi + cpu: 100m + +restorejob: + resources: + requests: + memory: 100Mi + cpu: 50m + limits: + memory: 200Mi + cpu: 100m diff --git a/examples/aurora/main.tf b/examples/aurora/main.tf index 0e08b98..5c59f3f 100644 --- a/examples/aurora/main.tf +++ b/examples/aurora/main.tf @@ -4,22 +4,25 @@ locals { external_id = "" # Define your external ID here assume_role_config = length(local.role_arn) > 0 ? { role_arn = local.role_arn } : null name = "skaf" - region = "us-east-2" - port = 5432 #/3306 - family = "aurora-postgresql15" #/aurora-mysql5.7" - engine = "aurora-postgresql" #/aurora-mysql" + region = "us-east-1" + port = 5432 # 3306 for MySQL + family = "aurora-postgresql15" # aurora-mysql5.7" + engine = "aurora-postgresql" # aurora-mysql" vpc_cidr = "10.0.0.0/16" - environment = "production" - db_engine_version = "15.2" #/5.7" + environment = "prod" + db_engine_version = "15.7" # 5.7" db_instance_class = "db.r5.large" - master_password = "" # Leave this field empty to have a password automatically generated. + cluster_name = "" + create_namespace = false + namespace = "mydb" + master_password = "" # Leave this field empty to have a password automatically generated. additional_aws_tags = { Owner = "Organization_Name" Expires = "Never" Department = "Engineering" } - current_identity = data.aws_caller_identity.current.arn - allowed_cidr_blocks = ["10.10.0.0/16"] + current_identity = data.aws_caller_identity.current.arn + allowed_cidr_blocks = ["10.0.0.0/16"] } data "aws_caller_identity" "current" {} @@ -86,7 +89,9 @@ module "vpc" { module "aurora" { source = "squareops/rds-aurora/aws" - version = "2.2.1" + version = "3.0.0" + name = local.name + region = local.region role_arn = local.role_arn external_id = local.external_id environment = local.environment @@ -123,4 +128,20 @@ module "aurora" { autoscaling_scale_in_cooldown = 60 autoscaling_scale_out_cooldown = 30 allowed_cidr_blocks = local.allowed_cidr_blocks + ######### + cluster_name = local.cluster_name # cluster name where your backup or restore job will run. + namespace = local.namespace + create_namespace = local.create_namespace + bucket_provider_type = "s3" + db_backup_enabled = false + db_backup_config = { + mysql_database_name = "atmosly_db4" # Specify the database name or Leave empty if you wish to backup all databases + cron_for_full_backup = "*/2 * * * *" # set cronjob for backup + bucket_uri = "s3://my-backup-dumps-databases" # S3 bucket URI (without a trailing slash /) + } + db_restore_enabled = false + db_restore_config = { + bucket_uri = "s3://my-backup-dumps-databases" # S3 bucket URI (without a trailing slash /) containing the backup dump file. + file_name = "atmosly_db1.sql" # Give .sql or .zip file for restore + } } diff --git a/examples/aurora/provider.tf b/examples/aurora/provider.tf index 67b50fa..50e7931 100644 --- a/examples/aurora/provider.tf +++ b/examples/aurora/provider.tf @@ -9,3 +9,25 @@ provider "aws" { } } } + +data "aws_eks_cluster" "cluster" { + name = local.cluster_name + +} +data "aws_eks_cluster_auth" "cluster" { + name = local.cluster_name +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token +} + +provider "helm" { + kubernetes { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + } +} diff --git a/helm/values/backup/values.yaml b/helm/values/backup/values.yaml new file mode 100644 index 0000000..4276d8f --- /dev/null +++ b/helm/values/backup/values.yaml @@ -0,0 +1,37 @@ +## Enable Full backup +backup: + bucket_uri: ${bucket_uri} + cron_for_full_backup: "${cron_for_full_backup}" + database_name: "${mysql_database_name}" + database_endpoint: "${db_endpoint}" + database_password: "${db_password}" + database_user: "${db_username}" + engine: "${engine}" + + +annotations: + ${annotations} + +auth: + username: "${custom_user_username}" + +bucket_provider_type: ${bucket_provider_type} + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Addons-Services" + operator: In + values: + - "true" + +backupjob: + resources: + requests: + memory: 100Mi + cpu: 50m + limits: + memory: 200Mi + cpu: 100m diff --git a/helm/values/restore/values.yaml b/helm/values/restore/values.yaml new file mode 100644 index 0000000..b98c684 --- /dev/null +++ b/helm/values/restore/values.yaml @@ -0,0 +1,35 @@ +restore: + file_name: ${file_name} + bucket_uri: ${bucket_uri} + database_endpoint: "${db_endpoint}" + database_password: "${db_password}" + database_user: "${db_username}" + engine: "${engine}" + # port: 5432 + +auth: + username: "${custom_user_username}" + +annotations: + ${annotations} + +bucket_provider_type: ${bucket_provider_type} + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Addons-Services" + operator: In + values: + - "true" + +restorejob: + resources: + requests: + memory: 100Mi + cpu: 50m + limits: + memory: 200Mi + cpu: 100m diff --git a/main.tf b/main.tf index 6f6f3ee..2d3034d 100644 --- a/main.tf +++ b/main.tf @@ -4,7 +4,7 @@ locals { Environment = var.environment } region = var.region - secondary_region = var.secondary_region + secondary_region = var.secondary_region != "null" ? var.secondary_region : null # Check if secondary_region is null role_arn = var.role_arn external_id = var.external_id assume_role_config = length(var.role_arn) > 0 ? { role_arn = var.role_arn } : null @@ -23,7 +23,7 @@ provider "aws" { provider "aws" { alias = "secondary" - region = local.secondary_region + region = local.secondary_region != null ? local.secondary_region : var.region # Fallback to primary region if secondary is null } data "aws_caller_identity" "current" {} @@ -36,7 +36,7 @@ module "aurora" { source = "terraform-aws-modules/rds-aurora/aws" version = "8.3.0" name = format("%s-%s", var.environment, var.rds_instance_name) - + # region = var.region engine = var.engine engine_mode = var.engine_mode engine_version = var.engine_mode == "serverless" ? null : var.engine_version @@ -63,7 +63,7 @@ module "aurora" { # security_groups = var.allowed_security_groups security_group_rules = { ingress_postgresql = { - description = "Allow inbound PostgreSQL traffic from trusted CIDR blocks" + description = "Allow inbound traffic from trusted CIDR blocks" type = "ingress" from_port = var.port to_port = var.port @@ -78,7 +78,6 @@ module "aurora" { protocol = "-1" cidr_blocks = ["0.0.0.0/0"] } -} subnets = var.subnets master_password = var.master_password != "" ? var.master_password : (length(random_password.master) > 0 ? random_password.master[0].result : null) @@ -161,7 +160,7 @@ resource "aws_rds_cluster_parameter_group" "rds_cluster_parameter_group" { } resource "aws_secretsmanager_secret" "secret_master_db" { - name = format("%s/%s/%s-%s", var.environment, var.rds_instance_name, var.engine, "aurora-pass") + name = format("%s/%s/%s-%s", var.environment, var.rds_instance_name, var.engine, "aurora-password") tags = merge( { "Name" = format("%s/%s/%s-%s", var.environment, var.rds_instance_name, var.engine, "aurora-pass") }, local.tags, @@ -196,10 +195,10 @@ resource "aws_rds_global_cluster" "this" { } module "aurora_secondary" { - source = "terraform-aws-modules/rds-aurora/aws" - count = var.global_cluster_enable ? 1 : 0 - version = "8.3.0" - providers = { aws = aws.secondary } + source = "terraform-aws-modules/rds-aurora/aws" + count = var.global_cluster_enable ? 1 : 0 + version = "8.3.0" + # providers = { aws = aws.secondary } is_primary_cluster = false @@ -239,3 +238,33 @@ module "aurora_secondary" { local.tags, ) } + + +module "backup_restore" { + depends_on = [module.aurora] + source = "./modules/db-backup-restore" + name = var.name + cluster_name = var.cluster_name + namespace = var.namespace + create_namespace = var.create_namespace + bucket_provider_type = var.bucket_provider_type + engine = var.engine + db_backup_enabled = var.db_backup_enabled + db_backup_config = { + db_username = module.aurora.cluster_master_username + db_password = var.master_password != "" ? var.master_password : nonsensitive(random_password.master[0].result) + mysql_database_name = var.db_backup_config.mysql_database_name + cron_for_full_backup = var.db_backup_config.cron_for_full_backup + bucket_uri = var.db_backup_config.bucket_uri + db_endpoint = module.aurora.cluster_endpoint + } + + db_restore_enabled = var.db_restore_enabled + db_restore_config = { + db_endpoint = module.aurora.cluster_endpoint + db_username = module.aurora.cluster_master_username + db_password = var.master_password != "" ? var.master_password : nonsensitive(random_password.master[0].result) + bucket_uri = var.db_restore_config.bucket_uri + file_name = var.db_restore_config.file_name + } +} diff --git a/modules/db-backup-restore/README.md b/modules/db-backup-restore/README.md new file mode 100644 index 0000000..01d3e3f --- /dev/null +++ b/modules/db-backup-restore/README.md @@ -0,0 +1,58 @@ +# db-backup-restore + + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | n/a | +| [helm](#provider\_helm) | n/a | +| [kubernetes](#provider\_kubernetes) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_role.mysql_backup_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.mysql_restore_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [helm_release.db_backup](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.db_restore](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubernetes_namespace.db](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_eks_cluster.kubernetes_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [azure\_container\_name](#input\_azure\_container\_name) | Azure container name | `string` | `""` | no | +| [azure\_storage\_account\_key](#input\_azure\_storage\_account\_key) | Azure storage account key | `string` | `""` | no | +| [azure\_storage\_account\_name](#input\_azure\_storage\_account\_name) | Azure storage account name | `string` | `""` | no | +| [bucket\_provider\_type](#input\_bucket\_provider\_type) | Choose what type of provider you want (s3, gcs) | `string` | `"s3"` | no | +| [cluster\_name](#input\_cluster\_name) | Specifies the name of the EKS cluster to deploy the MySQL application on. | `string` | `""` | no | +| [create\_namespace](#input\_create\_namespace) | Specify whether or not to create the namespace if it does not already exist. Set it to true to create the namespace. | `string` | `false` | no | +| [db\_backup\_config](#input\_db\_backup\_config) | configuration options for MySQL database backups. It includes properties such as the S3 bucket URI, the S3 bucket region, and the cron expression for full backups. | `map(string)` |
{
"bucket_uri": "",
"cron_for_full_backup": "",
"mysql_database_name": ""
}
| no | +| [db\_backup\_enabled](#input\_db\_backup\_enabled) | Specifies whether to enable backups for MySQL database. | `bool` | `false` | no | +| [db\_permission](#input\_db\_permission) | access | `bool` | `false` | no | +| [db\_restore\_config](#input\_db\_restore\_config) | Configuration options for restoring dump to the MySQL database. | `any` |
{
"bucket_uri": "",
"file_name": ""
}
| no | +| [db\_restore\_enabled](#input\_db\_restore\_enabled) | Specifies whether to enable restoring dump to the MySQL database. | `bool` | `false` | no | +| [engine](#input\_engine) | The name of the database engine to be used for this DB cluster | `string` | `"aurora"` | no | +| [iam\_role\_arn\_backup](#input\_iam\_role\_arn\_backup) | IAM role ARN for backup (AWS) | `string` | `""` | no | +| [iam\_role\_arn\_restore](#input\_iam\_role\_arn\_restore) | IAM role ARN for restore (AWS) | `string` | `""` | no | +| [name](#input\_name) | Name identifier for module to be added as suffix to resources | `string` | `"test"` | no | +| [namespace](#input\_namespace) | Name of the Kubernetes namespace where the MYSQL deployment will be deployed. | `string` | `"db"` | no | +| [service\_account\_backup](#input\_service\_account\_backup) | Service account for backup (GCP) | `string` | `""` | no | +| [service\_account\_restore](#input\_service\_account\_restore) | Service account for restore (GCP) | `string` | `""` | no | + +## Outputs + +No outputs. + diff --git a/modules/db-backup-restore/backup/.helmignore b/modules/db-backup-restore/backup/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/modules/db-backup-restore/backup/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/modules/db-backup-restore/backup/Chart.yaml b/modules/db-backup-restore/backup/Chart.yaml new file mode 100644 index 0000000..3394465 --- /dev/null +++ b/modules/db-backup-restore/backup/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A helm chart for Backup of mysql and stored in S3 +name: mysql-backup +version: 1.0.0 diff --git a/modules/db-backup-restore/backup/templates/cronjob.yaml b/modules/db-backup-restore/backup/templates/cronjob.yaml new file mode 100644 index 0000000..3445e8d --- /dev/null +++ b/modules/db-backup-restore/backup/templates/cronjob.yaml @@ -0,0 +1,36 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: backup-job +spec: + schedule: {{ .Values.backup.cron_for_full_backup | quote }} + concurrencyPolicy: Forbid + suspend: false + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 1 + + jobTemplate: + spec: + template: + spec: + affinity: {{ .Values.affinity | toYaml | nindent 10 }} + restartPolicy: OnFailure + imagePullSecrets: + - name: regcred + serviceAccountName: sa-db-backup + containers: + - name: backup-database + image: {{ if eq .Values.backup.engine "aurora-mysql" }}squareops01/rds-mysql-backup:v1{{ else }}squareops01/rds-postgresql-backup:v2{{ end }} + imagePullPolicy: Always + env: + - name: {{if eq .Values.backup.engine "aurora-mysql"}}MYSQL_HOST{{ else }}DB_HOST{{ end }} + value: {{ .Values.backup.database_endpoint }} + - name: {{if eq .Values.backup.engine "aurora-mysql"}}DATABASE{{ else }}DB_NAME{{ end }} + value: {{ .Values.backup.database_name }} + - name: {{if eq .Values.backup.engine "aurora-mysql"}}MYSQL_USER{{ else }}DB_USER{{ end }} + value: {{ .Values.backup.database_user }} + - name: {{if eq .Values.backup.engine "aurora-mysql"}}MYSQL_PASSWORD{{ else }}DB_PASSWORD{{ end }} + value: {{ .Values.backup.database_password }} + - name: {{if eq .Values.backup.engine "aurora-mysql"}}MYSQL_BUCKET_URI{{ else }}S3_BUCKET{{ end }} + value: {{ .Values.backup.bucket_uri }} + resources: {{ .Values.backupjob.resources | toYaml | nindent 12 }} diff --git a/modules/db-backup-restore/backup/templates/service_account.yaml b/modules/db-backup-restore/backup/templates/service_account.yaml new file mode 100644 index 0000000..d43c007 --- /dev/null +++ b/modules/db-backup-restore/backup/templates/service_account.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sa-db-backup + namespace: {{ .Release.Namespace }} + annotations: + {{ toYaml .Values.annotations | indent 4 }} diff --git a/modules/db-backup-restore/main.tf b/modules/db-backup-restore/main.tf new file mode 100644 index 0000000..32f801e --- /dev/null +++ b/modules/db-backup-restore/main.tf @@ -0,0 +1,60 @@ +resource "kubernetes_namespace" "db" { + count = var.create_namespace ? 1 : 0 + metadata { + annotations = {} + name = var.namespace + } +} + +resource "helm_release" "db_backup" { + count = var.db_backup_enabled ? 1 : 0 + depends_on = [kubernetes_namespace.db] + name = "db-backup" + chart = "${path.module}/../../modules/db-backup-restore/backup" + timeout = 600 + namespace = var.namespace + values = [ + templatefile("${path.module}/../../helm/values/backup/values.yaml", { + bucket_uri = var.db_backup_config.bucket_uri, + engine = var.engine, + mysql_database_name = var.bucket_provider_type == "s3" ? var.db_backup_config.mysql_database_name : "", + db_endpoint = var.bucket_provider_type == "s3" ? var.db_backup_config.db_endpoint : "", + db_password = var.bucket_provider_type == "s3" ? var.db_backup_config.db_password : "", + db_username = var.bucket_provider_type == "s3" ? var.db_backup_config.db_username : "", + cron_for_full_backup = var.db_backup_config.cron_for_full_backup, + custom_user_username = "admin", + bucket_provider_type = var.bucket_provider_type, + azure_storage_account_name = var.bucket_provider_type == "azure" ? var.azure_storage_account_name : "" + azure_storage_account_key = var.bucket_provider_type == "azure" ? var.azure_storage_account_key : "" + azure_container_name = var.bucket_provider_type == "azure" ? var.azure_container_name : "" + annotations = var.bucket_provider_type == "s3" ? "eks.amazonaws.com/role-arn: ${aws_iam_role.mysql_backup_role[count.index].arn}" : "iam.gke.io/gcp-service-account: ${var.service_account_backup}" + }) + ] +} + + +## DB dump restore +resource "helm_release" "db_restore" { + count = var.db_restore_enabled ? 1 : 0 + depends_on = [kubernetes_namespace.db] + name = "db-restore" + chart = "${path.module}/../../modules/db-backup-restore/restore" + timeout = 600 + namespace = var.namespace + values = [ + templatefile("${path.module}/../../helm/values/restore/values.yaml", { + bucket_uri = var.db_restore_config.bucket_uri, + file_name = var.db_restore_config.file_name, + engine = var.engine, + db_endpoint = var.bucket_provider_type == "s3" ? var.db_restore_config.db_endpoint : "", + db_password = var.bucket_provider_type == "s3" ? var.db_restore_config.db_password : "", + db_username = var.bucket_provider_type == "s3" ? var.db_restore_config.db_username : "", + custom_user_username = "admin", + bucket_provider_type = var.bucket_provider_type, + azure_storage_account_name = var.bucket_provider_type == "azure" ? var.azure_storage_account_name : "" + azure_storage_account_key = var.bucket_provider_type == "azure" ? var.azure_storage_account_key : "" + azure_container_name = var.bucket_provider_type == "azure" ? var.azure_container_name : "" + annotations = var.bucket_provider_type == "s3" ? "eks.amazonaws.com/role-arn: ${aws_iam_role.mysql_restore_role[count.index].arn}" : "iam.gke.io/gcp-service-account: ${var.service_account_restore}" + }) + ] +} diff --git a/modules/db-backup-restore/restore/.helmignore b/modules/db-backup-restore/restore/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/modules/db-backup-restore/restore/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/modules/db-backup-restore/restore/Chart.yaml b/modules/db-backup-restore/restore/Chart.yaml new file mode 100644 index 0000000..661e06b --- /dev/null +++ b/modules/db-backup-restore/restore/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A helm chart for restore of mysql and stored in S3 +name: mysql-restore +version: 1.0.0 diff --git a/modules/db-backup-restore/restore/templates/job.yaml b/modules/db-backup-restore/restore/templates/job.yaml new file mode 100644 index 0000000..423844a --- /dev/null +++ b/modules/db-backup-restore/restore/templates/job.yaml @@ -0,0 +1,49 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: restore +spec: + template: + spec: + affinity: {{ .Values.affinity | toYaml | nindent 6 }} + serviceAccountName: sa-db-restore + containers: + - name: restore-job + image: {{ if eq .Values.restore.engine "aurora-mysql" }}squareops01/rds-mysql-restore:v1{{ else }}squareops01/rds-postgresql-restore:v2{{ end }} + imagePullPolicy: Always + env: + - name: {{if eq .Values.restore.engine "aurora-mysql"}}MYSQL_HOST{{ else }}DB_HOST{{ end }} + value: {{ .Values.restore.database_endpoint }} + - name: {{if eq .Values.restore.engine "aurora-mysql"}}DATABASE{{ else }}DB_NAME{{ end }} + value: {{ .Values.restore.database_name }} + - name: {{if eq .Values.restore.engine "aurora-mysql"}}MYSQL_USER{{ else }}DB_USER{{ end }} + value: {{ .Values.restore.database_user }} + - name: {{if eq .Values.restore.engine "aurora-mysql"}}MYSQL_PASSWORD{{ else }}DB_PASSWORD{{ end }} + value: {{ .Values.restore.database_password }} + - name: {{if eq .Values.restore.engine "aurora-mysql"}}MYSQL_BUCKET_RESTORE_URI{{ else }}POSTGRESQL_BUCKET_RESTORE_URI{{ end }} + value: {{ .Values.restore.bucket_uri }} + - name: {{if eq .Values.restore.engine "aurora-mysql"}}RESTORE_FILE_NAME{{ else }}RESTORE_FILE_NAME{{ end }} + value: {{ .Values.restore.file_name}} + resources: {{ .Values.restorejob.resources | toYaml | nindent 12 }} + {{- if eq .Values.restore.engine "aurora-mysql" }} + initContainers: + - name: grant-system-variable-admin + image: mysql:8.0 + command: + - /bin/bash + - -c + - | + # Connect to the MySQL server using the MySQL client + mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO '$MYSQL_USER'@'%'; FLUSH PRIVILEGES;" + # Exit with status code 0 to indicate success + exit 0 + env: + - name: MYSQL_HOST + value: {{ .Values.restore.database_endpoint }} + - name: MYSQL_USER + value: {{ .Values.restore.database_user }} + - name: MYSQL_PASSWORD + value: {{ .Values.restore.database_password }} + {{- end }} + restartPolicy: Never + backoffLimit: 4 diff --git a/modules/db-backup-restore/restore/templates/restore-secret.yaml b/modules/db-backup-restore/restore/templates/restore-secret.yaml new file mode 100644 index 0000000..8ead489 --- /dev/null +++ b/modules/db-backup-restore/restore/templates/restore-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: mysql-bucket-uri-restore + namespace: {{ .Release.Namespace }} + labels: +data: + MYSQL_BUCKET_URI: {{ .Values.restore.bucket_uri | b64enc | quote }} diff --git a/modules/db-backup-restore/restore/templates/service_account.yaml b/modules/db-backup-restore/restore/templates/service_account.yaml new file mode 100644 index 0000000..8b4e659 --- /dev/null +++ b/modules/db-backup-restore/restore/templates/service_account.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sa-db-restore + annotations: + {{ toYaml .Values.annotations | indent 4 }} diff --git a/modules/db-backup-restore/roles.tf b/modules/db-backup-restore/roles.tf new file mode 100644 index 0000000..617c0cc --- /dev/null +++ b/modules/db-backup-restore/roles.tf @@ -0,0 +1,100 @@ +locals { + oidc_provider = replace( + data.aws_eks_cluster.kubernetes_cluster.identity[0].oidc[0].issuer, + "/^https:///", + "" + ) +} + +data "aws_caller_identity" "current" {} + +data "aws_eks_cluster" "kubernetes_cluster" { + name = var.cluster_name +} + +resource "aws_iam_role" "mysql_backup_role" { + count = var.db_backup_enabled ? 1 : 0 + name = format("%s-%s-%s", var.cluster_name, var.name, "db-backup") + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Principal = { + Federated = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.oidc_provider}" + }, + Action = "sts:AssumeRoleWithWebIdentity", + Condition = { + StringEquals = { + "${local.oidc_provider}:aud" = "sts.amazonaws.com", + "${local.oidc_provider}:sub" = "system:serviceaccount:${var.namespace}:sa-db-backup" + } + } + } + ] + }) + inline_policy { + name = "AllowS3PutObject" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ] + Effect = "Allow" + Resource = "*" + } + ] + }) + } +} + + +resource "aws_iam_role" "mysql_restore_role" { + count = var.db_restore_enabled ? 1 : 0 + name = format("%s-%s-%s", var.cluster_name, var.name, "db-restore") + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Principal = { + Federated = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.oidc_provider}" + }, + Action = "sts:AssumeRoleWithWebIdentity", + Condition = { + StringEquals = { + "${local.oidc_provider}:aud" = "sts.amazonaws.com", + "${local.oidc_provider}:sub" = "system:serviceaccount:${var.namespace}:sa-db-restore" + } + } + } + ] + }) + inline_policy { + name = "AllowS3PutObject" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ] + Effect = "Allow" + Resource = "*" + } + ] + }) + } +} diff --git a/modules/db-backup-restore/variables.tf b/modules/db-backup-restore/variables.tf new file mode 100644 index 0000000..b8042ba --- /dev/null +++ b/modules/db-backup-restore/variables.tf @@ -0,0 +1,119 @@ +variable "iam_role_arn_backup" { + description = "IAM role ARN for backup (AWS)" + type = string + default = "" +} + +variable "service_account_backup" { + description = "Service account for backup (GCP)" + type = string + default = "" +} + +variable "azure_storage_account_name" { + description = "Azure storage account name" + type = string + default = "" +} + +variable "azure_storage_account_key" { + description = "Azure storage account key" + type = string + default = "" +} + +variable "azure_container_name" { + description = "Azure container name" + type = string + default = "" +} + +variable "namespace" { + type = string + default = "db" + description = "Name of the Kubernetes namespace where the MYSQL deployment will be deployed." +} + +variable "create_namespace" { + type = string + description = "Specify whether or not to create the namespace if it does not already exist. Set it to true to create the namespace." + default = false +} + +variable "iam_role_arn_restore" { + description = "IAM role ARN for restore (AWS)" + type = string + default = "" +} + +variable "service_account_restore" { + description = "Service account for restore (GCP)" + type = string + default = "" +} + +# two variable of clustername and name +variable "name" { + description = "Name identifier for module to be added as suffix to resources" + type = string + default = "test" +} + +variable "cluster_name" { + type = string + default = "" + description = "Specifies the name of the EKS cluster to deploy the MySQL application on." +} + +variable "db_permission" { + default = false + description = "access" + type = bool +} + +variable "bucket_provider_type" { + type = string + default = "s3" + description = "Choose what type of provider you want (s3, gcs)" +} + + +variable "db_backup_enabled" { + type = bool + default = false + description = "Specifies whether to enable backups for MySQL database." +} + +variable "db_restore_enabled" { + type = bool + default = false + description = "Specifies whether to enable restoring dump to the MySQL database." +} + +variable "db_backup_config" { + type = map(string) + default = { + bucket_uri = "" + # s3_bucket_region = "" + cron_for_full_backup = "" + mysql_database_name = "" + # port = "" + } + description = "configuration options for MySQL database backups. It includes properties such as the S3 bucket URI, the S3 bucket region, and the cron expression for full backups." +} + +variable "db_restore_config" { + type = any + default = { + bucket_uri = "" + file_name = "" + # s3_bucket_region = "" + } + description = "Configuration options for restoring dump to the MySQL database." +} + +variable "engine" { + description = "The name of the database engine to be used for this DB cluster" + type = string + default = "aurora" +} diff --git a/outputs.tf b/outputs.tf index ad98641..d348ceb 100644 --- a/outputs.tf +++ b/outputs.tf @@ -18,7 +18,7 @@ output "secondary_rds_cluster_reader_endpoint" { value = var.global_cluster_enable ? module.aurora_secondary[0].cluster_reader_endpoint : null } -output "rds_cluster_database_name"{ +output "rds_cluster_database_name" { description = "Name for an automatically created database on cluster creation" value = module.aurora.cluster_database_name } diff --git a/variables.tf b/variables.tf index 647b0b0..bc59db6 100644 --- a/variables.tf +++ b/variables.tf @@ -1,3 +1,8 @@ +variable "name" { + description = "The name of the RDS instance" + default = "" + type = string +} variable "allowed_cidr_blocks" { description = "A list of CIDR blocks which are allowed to access the database" type = any @@ -389,3 +394,62 @@ variable "external_id" { type = string default = "" # Default to empty string if not provided } + +### backup & restore + +variable "cluster_name" { + type = string + default = "" + description = "Specifies the name of the EKS cluster to deploy the MySQL application on." +} + +variable "create_namespace" { + type = string + description = "Specify whether or not to create the namespace if it does not already exist. Set it to true to create the namespace." + default = false +} + +variable "namespace" { + type = string + default = "db" + description = "Name of the Kubernetes namespace where the MYSQL deployment will be deployed." +} + +variable "db_backup_enabled" { + type = bool + default = false + description = "Specifies whether to enable backups for MySQL database." +} + +variable "db_restore_enabled" { + type = bool + default = false + description = "Specifies whether to enable restoring dump to the MySQL database." +} +variable "bucket_provider_type" { + type = string + default = "s3" + description = "Choose what type of provider you want (s3, gcs)" +} + +variable "db_backup_config" { + type = map(string) + default = { + bucket_uri = "" + # s3_bucket_region = "" + cron_for_full_backup = "" + mysql_database_name = "" + # port ="" + } + description = "configuration options for MySQL database backups. It includes properties such as the S3 bucket URI, the S3 bucket region, and the cron expression for full backups." +} + +variable "db_restore_config" { + type = any + default = { + bucket_uri = "" + file_name = "" + # s3_bucket_region = "" + } + description = "Configuration options for restoring dump to the MySQL database." +}