diff --git a/cmd/config.go b/cmd/config.go index fff5f72..7922129 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -65,10 +65,11 @@ type GlobalConfig struct { } type FeatureConfig struct { - Scuba ScubaFeatureConfig `yaml:"scuba"` - BucketNotifications BucketNotificationsFeatureConfig `yaml:"bucket_notifications"` - Utapi UtapiFeatureConfig `yaml:"utapi"` - Migration MigrationFeatureConfig `yaml:"migration"` + Scuba ScubaFeatureConfig `yaml:"scuba"` + BucketNotifications BucketNotificationsFeatureConfig `yaml:"bucket_notifications"` + CrossRegionReplication CrossRegionReplicationFeatureConfig `yaml:"cross_region_replication"` + Utapi UtapiFeatureConfig `yaml:"utapi"` + Migration MigrationFeatureConfig `yaml:"migration"` } type ScubaFeatureConfig struct { @@ -85,6 +86,10 @@ type BucketNotificationsFeatureConfig struct { } `yaml:"destinationAuth"` } +type CrossRegionReplicationFeatureConfig struct { + Enabled bool `yaml:"enabled"` +} + type UtapiFeatureConfig struct { Enabled bool `yaml:"enabled"` } @@ -229,6 +234,9 @@ func DefaultEnvironmentConfig() EnvironmentConfig { Migration: MigrationFeatureConfig{ Enabled: false, }, + CrossRegionReplication: CrossRegionReplicationFeatureConfig{ + Enabled: false, + }, }, Cloudserver: CloudserverConfig{}, S3Metadata: MetadataConfig{ diff --git a/cmd/util.go b/cmd/util.go index d02e74f..416f0d4 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -84,6 +84,10 @@ func getComposeProfiles(cfg EnvironmentConfig) []string { profiles = append(profiles, "feature-migration") } + if cfg.Features.CrossRegionReplication.Enabled { + profiles = append(profiles, "feature-crr") + } + return profiles } diff --git a/templates/backbeat/config.json b/templates/backbeat/config.json index 4e12724..48af427 100644 --- a/templates/backbeat/config.json +++ b/templates/backbeat/config.json @@ -1,28 +1,45 @@ { "zookeeper": { "connectionString": "127.0.0.1:2181/backbeat", - "autoCreateNamespace": true + "autoCreateNamespace": false }, "kafka": { "hosts": "127.0.0.1:9092", + "compressionType": "none", + "requiredAcks": 1, "backlogMetrics": { "zkPath": "/run/kafka-backlog-metrics", "intervalS": 60 - } + }, + "maxRequestSize": 5000020 }, "s3": { "host": "127.0.0.1", "port": 8000 }, + "vaultAdmin": { + "host": "127.0.0.1", + "port": 8500 + }, + "replicationGroupId": "RG001 ", "queuePopulator": { "cronRule": "*/5 * * * * *", "batchMaxRead": 10000, + "batchTimeoutMs": 9000, "zookeeperPath": "/queue-populator", "logSource": "bucketd", "bucketd": { "host": "127.0.0.1", "port": 9000 }, + "dmd": { + "host": "127.0.0.1", + "port": 9990 + }, + "kafka": { + "topic": "backbeat-oplog", + "consumerGroupId": "backbeat-qp-oplog-group" + }, "probeServer": { "bindAddress": "localhost", "port": 4042 @@ -37,27 +54,32 @@ "port": 8000 }, "auth": { - "type": "account", - "account": "bart", + "type": "role", "vault": { "host": "127.0.0.1", "port": 8500, - "adminPort": 8600 + "adminPort": 8600, + "adminCredentialsFile": "/home/scality/backbeat/node_modules/vaultclient/tests/utils/admincredentials.json" } } }, "destination": { "transport": "http", "bootstrapList": [ - { "site": "sf", "servers": ["127.0.0.1:9443"], "echo": false }, - { "site": "aws-location", "type": "aws_s3" } + { "site": "sf", "servers": ["127.0.0.1:8000"], "echo": false } ], "auth": { - "type": "account", - "account": "lisa" + "type": "role", + "vault": { + "host": "127.0.0.1", + "port": 8500, + "adminPort": 8600, + "adminCredentialsFile": "/home/scality/backbeat/node_modules/vaultclient/tests/utils/admincredentials.json" + } } }, "topic": "backbeat-replication", + "dataMoverTopic": "backbeat-data-mover", "replicationStatusTopic": "backbeat-replication-status", "replicationFailedTopic": "backbeat-replication-failed", "monitorReplicationFailures": true, @@ -70,26 +92,38 @@ ], "queueProcessor": { "groupId": "backbeat-replication-group", - "retryTimeoutS": 300, + "retry": { + "scality": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + } + }, "concurrency": 10, "mpuPartsConcurrency": 10, - "probeServer": [ - { - "site": "aws-location", - "bindAddress": "localhost", - "port": "4043" - }, { - "site" : "us-east-1", - "bindAddress": "localhost", - "port": "4044" - } - ] + "probeServer": { + "bindAddress": "localhost", + "port": 4043 + } }, "replicationStatusProcessor": { "groupId": "backbeat-replication-group", - "retryTimeoutS": 300, + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, "concurrency": 10, - "logConsumerMetricsIntervalS": 60, "probeServer": { "bindAddress": "localhost", "port": 4045 @@ -137,7 +171,11 @@ "conductor": { "backlogControl": { "enabled": true }, "cronRule": "*/5 * * * * *", - "concurrency": 10 + "concurrency": 10, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8552 + } }, "bucketProcessor": { "groupId": "backbeat-lifecycle-bucket-processor-group", @@ -151,7 +189,11 @@ "factor": 1.5 } }, - "concurrency": 10 + "concurrency": 10, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8553 + } }, "objectProcessor": { "groupId": "backbeat-lifecycle-object-processor-group", @@ -165,7 +207,11 @@ "factor": 1.5 } }, - "concurrency": 10 + "concurrency": 10, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8554 + } } } }, diff --git a/templates/backbeat/env b/templates/backbeat/env index 52116c4..f29ef16 100644 --- a/templates/backbeat/env +++ b/templates/backbeat/env @@ -1 +1,3 @@ -EXPIRE_ONE_DAY_EARLIER=true +export EXPIRE_ONE_DAY_EARLIER=true +export CRASH_ON_BATCH_TIMEOUT=true +export REMOTE_MANAGEMENT_DISABLE=true diff --git a/templates/backbeat/supervisord.conf b/templates/backbeat/supervisord.conf index c486f79..11ffee5 100644 --- a/templates/backbeat/supervisord.conf +++ b/templates/backbeat/supervisord.conf @@ -49,3 +49,47 @@ stderr_logfile_backups=7 autorestart = true autostart = true {{ end }} + +## Cross Region Replication +{{ if .Features.CrossRegionReplication.Enabled }} +[program:crr-queue-populator] +command = bash -c "source /conf/env && exec npm run queue_populator" +numprocs = 1 +process_name = %(program_name)s_%(process_num)s +stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log +stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log +stdout_logfile_maxbytes=100MB +stdout_logfile_backups=7 +stderr_logfile_maxbytes=100MB +stderr_logfile_backups=7 +autorestart = true +autostart = true + +[program:crr-queue-processor] +command = bash -c "source /conf/env && exec npm run queue_processor" +environment = BOOTSTRAP_SITE_NAME=sf +numprocs = 1 +process_name = %(program_name)s_%(process_num)s +stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log +stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log +stdout_logfile_maxbytes=100MB +stdout_logfile_backups=7 +stderr_logfile_maxbytes=100MB +stderr_logfile_backups=7 +autorestart = true +autostart = true + +[program:crr-status-processor] +command = bash -c "source /conf/env && exec npm run replication_status_processor" +numprocs = 1 +process_name = %(program_name)s_%(process_num)s +stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log +stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log +stdout_logfile_maxbytes=100MB +stdout_logfile_backups=7 +stderr_logfile_maxbytes=100MB +stderr_logfile_backups=7 +autorestart = true +autostart = true + +{{ end }} diff --git a/templates/cloudserver/config-v7.json b/templates/cloudserver/config-v7.json index 858780f..49493cd 100644 --- a/templates/cloudserver/config-v7.json +++ b/templates/cloudserver/config-v7.json @@ -1,6 +1,7 @@ { "port": 8000, "listenOn": [], + "metricsPort": 8002, "replicationGroupId": "RG001", "restEndpoints": { "localhost": "us-east-1", @@ -28,7 +29,7 @@ ], "replicationEndpoints": [ { - "site": "zenko", + "site": "sf", "servers": ["127.0.0.1:8000"], "default": true }, diff --git a/templates/cloudserver/config-v9.json b/templates/cloudserver/config-v9.json index ee461d1..2722330 100644 --- a/templates/cloudserver/config-v9.json +++ b/templates/cloudserver/config-v9.json @@ -33,7 +33,7 @@ ], "replicationEndpoints": [ { - "site": "zenko", + "site": "sf", "servers": ["127.0.0.1:8000"], "default": true }, diff --git a/templates/global/docker-compose.yaml b/templates/global/docker-compose.yaml index cb6d821..6095651 100644 --- a/templates/global/docker-compose.yaml +++ b/templates/global/docker-compose.yaml @@ -214,8 +214,16 @@ services: network_mode: host environment: KAFKA_PORT: '9092' - TOPICS_TO_CREATE: backbeat-lifecycle-bucket-tasks backbeat-lifecycle-object-tasks backbeat-bucket-notification - CREATE_NOTIFICATION_PATHS: 'true' + TOPICS_TO_CREATE: > + backbeat-lifecycle-bucket-tasks + backbeat-lifecycle-object-tasks + backbeat-bucket-notification + backbeat-replication + backbeat-data-mover + backbeat-replication-status + backbeat-replication-failed + backbeat-metrics-group-crr + CREATE_ZOOKEEPER_PATHS: 'true' ZOOKEEPER_ENDPOINT: 127.0.0.1:2181/backbeat depends_on: setup-vault: diff --git a/templates/global/values.yaml b/templates/global/values.yaml index baeb15e..d94a4a4 100644 --- a/templates/global/values.yaml +++ b/templates/global/values.yaml @@ -13,6 +13,9 @@ features: username: admin password: admin123 + cross_region_replication: + enabled: false + utapi: enabled: false diff --git a/templates/kafka/setup.sh b/templates/kafka/setup.sh index 82d1dda..130f49f 100644 --- a/templates/kafka/setup.sh +++ b/templates/kafka/setup.sh @@ -39,7 +39,7 @@ if [[ -n "$TOPICS_TO_CREATE" ]]; then done fi -if [[ "$CREATE_NOTIFICATION_PATHS" == "true" ]]; then +if [[ "$CREATE_ZOOKEEPER_PATHS" == "true" ]]; then if [[ -z "$ZOOKEEPER_ENDPOINT" ]]; then echo "[setup] Zookeeper endpoint not set" exit 1 @@ -56,6 +56,14 @@ create /bucket-notification/raft-id-dispatcher/provisions create /bucket-notification/raft-id-dispatcher/provisions/1 create /bucket-notification/raft-id-dispatcher/provisions/2 create /bucket-notification/raft-id-dispatcher/provisions/3 +create /queue-populator +create /queue-populator/raft-id-dispatcher +create /queue-populator/raft-id-dispatcher/owners +create /queue-populator/raft-id-dispatcher/leaders +create /queue-populator/raft-id-dispatcher/provisions +create /queue-populator/raft-id-dispatcher/provisions/0 +create /queue-populator/raft-id-dispatcher/provisions/1 +create /queue-populator/raft-id-dispatcher/provisions/2 quit EOF echo "[setup] Zookeeper paths created."