WKBCH-12: Add crr support

cmd/config.go

@@ -65,10 +65,11 @@ type GlobalConfig struct {
 }
 
 type FeatureConfig struct {
-	Scuba               ScubaFeatureConfig               `yaml:"scuba"`
-	BucketNotifications BucketNotificationsFeatureConfig `yaml:"bucket_notifications"`
-	Utapi               UtapiFeatureConfig               `yaml:"utapi"`
-	Migration           MigrationFeatureConfig           `yaml:"migration"`
+	Scuba                  ScubaFeatureConfig               `yaml:"scuba"`
+	BucketNotifications    BucketNotificationsFeatureConfig `yaml:"bucket_notifications"`
+	CrossRegionReplication CrossRegionReplicationFeatureConfig `yaml:"cross_region_replication"`
+	Utapi                  UtapiFeatureConfig               `yaml:"utapi"`
+	Migration              MigrationFeatureConfig           `yaml:"migration"`
 }
 
 type ScubaFeatureConfig struct {
@@ -85,6 +86,10 @@ type BucketNotificationsFeatureConfig struct {
 	} `yaml:"destinationAuth"`
 }
 
+type CrossRegionReplicationFeatureConfig struct {
+	Enabled bool `yaml:"enabled"`
+}
+
 type UtapiFeatureConfig struct {
 	Enabled bool `yaml:"enabled"`
 }
@@ -229,6 +234,9 @@ func DefaultEnvironmentConfig() EnvironmentConfig {
 			Migration: MigrationFeatureConfig{
 				Enabled: false,
 			},
+			CrossRegionReplication: CrossRegionReplicationFeatureConfig{
+				Enabled: false,
+			},
 		},
 		Cloudserver: CloudserverConfig{},
 		S3Metadata: MetadataConfig{

cmd/util.go

@@ -84,6 +84,10 @@ func getComposeProfiles(cfg EnvironmentConfig) []string {
 		profiles = append(profiles, "feature-migration")
 	}
 
+	if cfg.Features.CrossRegionReplication.Enabled {
+		profiles = append(profiles, "feature-crr")
+	}
+
 	return profiles
 }
 

templates/backbeat/config.json

@@ -1,28 +1,45 @@
 {
     "zookeeper": {
         "connectionString": "127.0.0.1:2181/backbeat",
-        "autoCreateNamespace": true
+        "autoCreateNamespace": false
     },
     "kafka": {
         "hosts": "127.0.0.1:9092",
+        "compressionType": "none",
+        "requiredAcks": 1,
         "backlogMetrics": {
             "zkPath": "/run/kafka-backlog-metrics",
             "intervalS": 60
-        }
+        },
+        "maxRequestSize": 5000020
     },
     "s3": {
         "host": "127.0.0.1",
         "port": 8000
     },
+    "vaultAdmin": {
+        "host": "127.0.0.1",
+        "port": 8500
+    },
+    "replicationGroupId": "RG001  ",
     "queuePopulator": {
         "cronRule": "*/5 * * * * *",
         "batchMaxRead": 10000,
+        "batchTimeoutMs": 9000,
         "zookeeperPath": "/queue-populator",
         "logSource": "bucketd",
         "bucketd": {
             "host": "127.0.0.1",
             "port": 9000
         },
+        "dmd": {
+            "host": "127.0.0.1",
+            "port": 9990
+        },
+        "kafka": {
+            "topic": "backbeat-oplog",
+            "consumerGroupId": "backbeat-qp-oplog-group"
+        },
         "probeServer": {
             "bindAddress": "localhost",
             "port": 4042
@@ -37,27 +54,32 @@
                     "port": 8000
                 },
                 "auth": {
-                    "type": "account",
-                    "account": "bart",
+                    "type": "role",
                     "vault": {
                         "host": "127.0.0.1",
                         "port": 8500,
-                        "adminPort": 8600
+                        "adminPort": 8600,
+                        "adminCredentialsFile": "/home/scality/backbeat/node_modules/vaultclient/tests/utils/admincredentials.json"
                     }
                 }
             },
             "destination": {
                 "transport": "http",
                 "bootstrapList": [
-                    { "site": "sf", "servers": ["127.0.0.1:9443"], "echo": false },
-                    { "site": "aws-location", "type": "aws_s3" }
+                    { "site": "sf", "servers": ["127.0.0.1:8000"], "echo": false }
                 ],
                 "auth": {
-                    "type": "account",
-                    "account": "lisa"
+                    "type": "role",
+                    "vault": {
+                        "host": "127.0.0.1",
+                        "port": 8500,
+                        "adminPort": 8600,
+                        "adminCredentialsFile": "/home/scality/backbeat/node_modules/vaultclient/tests/utils/admincredentials.json"
+                    }
                 }
             },
             "topic": "backbeat-replication",
+            "dataMoverTopic": "backbeat-data-mover",
             "replicationStatusTopic": "backbeat-replication-status",
             "replicationFailedTopic": "backbeat-replication-failed",
             "monitorReplicationFailures": true,
@@ -70,26 +92,38 @@
             ],
             "queueProcessor": {
                 "groupId": "backbeat-replication-group",
-                "retryTimeoutS": 300,
+                "retry": {
+                    "scality": {
+                        "maxRetries": 5,
+                        "timeoutS": 300,
+                        "backoff": {
+                            "min": 1000,
+                            "max": 300000,
+                            "jitter": 0.1,
+                            "factor": 1.5
+                        }
+                    }
+                },
                 "concurrency": 10,
                 "mpuPartsConcurrency": 10,
-                "probeServer": [
-                    {
-                        "site": "aws-location",
-                        "bindAddress": "localhost",
-                        "port": "4043"
-                    }, {
-                        "site" : "us-east-1",
-                        "bindAddress": "localhost",
-                        "port": "4044"
-                    }
-                ]
+                "probeServer": {
+                    "bindAddress": "localhost",
+                    "port": 4043
+                }
             },
             "replicationStatusProcessor": {
                 "groupId": "backbeat-replication-group",
-                "retryTimeoutS": 300,
+                "retry": {
+                    "maxRetries": 5,
+                    "timeoutS": 300,
+                    "backoff": {
+                        "min": 1000,
+                        "max": 300000,
+                        "jitter": 0.1,
+                        "factor": 1.5
+                    }
+                },
                 "concurrency": 10,
-                "logConsumerMetricsIntervalS": 60,
                 "probeServer": {
                     "bindAddress": "localhost",
                     "port": 4045
@@ -137,7 +171,11 @@
             "conductor": {
                 "backlogControl": { "enabled": true },
                 "cronRule": "*/5 * * * * *",
-                "concurrency": 10
+                "concurrency": 10,
+                "probeServer": {
+                    "bindAddress": "0.0.0.0",
+                    "port": 8552
+                }
             },
             "bucketProcessor": {
                 "groupId": "backbeat-lifecycle-bucket-processor-group",
@@ -151,7 +189,11 @@
                         "factor": 1.5
                     }
                 },
-                "concurrency": 10
+                "concurrency": 10,
+                "probeServer": {
+                    "bindAddress": "0.0.0.0",
+                    "port": 8553
+                }
             },
             "objectProcessor": {
                 "groupId": "backbeat-lifecycle-object-processor-group",
@@ -165,7 +207,11 @@
                         "factor": 1.5
                     }
                 },
-                "concurrency": 10
+                "concurrency": 10,
+                "probeServer": {
+                    "bindAddress": "0.0.0.0",
+                    "port": 8554
+                }
             }
         }
     },

templates/backbeat/env

@@ -1 +1,3 @@
-EXPIRE_ONE_DAY_EARLIER=true
+export EXPIRE_ONE_DAY_EARLIER=true
+export CRASH_ON_BATCH_TIMEOUT=true
+export REMOTE_MANAGEMENT_DISABLE=true

templates/backbeat/supervisord.conf

@@ -49,3 +49,47 @@ stderr_logfile_backups=7
 autorestart = true
 autostart = true
 {{ end }}
+
+## Cross Region Replication
+{{ if .Features.CrossRegionReplication.Enabled }}
+[program:crr-queue-populator]
+command = bash -c "source /conf/env && exec npm run queue_populator"
+numprocs = 1
+process_name = %(program_name)s_%(process_num)s
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+stderr_logfile_maxbytes=100MB
+stderr_logfile_backups=7
+autorestart = true
+autostart = true
+
+[program:crr-queue-processor]
+command = bash -c "source /conf/env && exec npm run queue_processor"
+environment = BOOTSTRAP_SITE_NAME=sf
+numprocs = 1
+process_name = %(program_name)s_%(process_num)s
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+stderr_logfile_maxbytes=100MB
+stderr_logfile_backups=7
+autorestart = true
+autostart = true
+
+[program:crr-status-processor]
+command = bash -c "source /conf/env && exec npm run replication_status_processor"
+numprocs = 1
+process_name = %(program_name)s_%(process_num)s
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+stderr_logfile_maxbytes=100MB
+stderr_logfile_backups=7
+autorestart = true
+autostart = true
+
+{{ end }}

templates/cloudserver/config-v7.json

@@ -1,6 +1,7 @@
 {
     "port": 8000,
     "listenOn": [],
+    "metricsPort": 8002,
     "replicationGroupId": "RG001",
     "restEndpoints": {
         "localhost": "us-east-1",
@@ -28,7 +29,7 @@
     ],
     "replicationEndpoints": [
         {
-            "site": "zenko",
+            "site": "sf",
             "servers": ["127.0.0.1:8000"],
             "default": true
         },

templates/cloudserver/config-v9.json

@@ -33,7 +33,7 @@
     ],
     "replicationEndpoints": [
         {
-            "site": "zenko",
+            "site": "sf",
             "servers": ["127.0.0.1:8000"],
             "default": true
         },

templates/global/docker-compose.yaml

@@ -214,8 +214,16 @@ services:
     network_mode: host
     environment:
       KAFKA_PORT: '9092'
-      TOPICS_TO_CREATE: backbeat-lifecycle-bucket-tasks backbeat-lifecycle-object-tasks backbeat-bucket-notification
-      CREATE_NOTIFICATION_PATHS: 'true'
+      TOPICS_TO_CREATE: >
+        backbeat-lifecycle-bucket-tasks
+        backbeat-lifecycle-object-tasks
+        backbeat-bucket-notification
+        backbeat-replication
+        backbeat-data-mover
+        backbeat-replication-status
+        backbeat-replication-failed
+        backbeat-metrics-group-crr
+      CREATE_ZOOKEEPER_PATHS: 'true'
       ZOOKEEPER_ENDPOINT: 127.0.0.1:2181/backbeat
     depends_on:
       setup-vault:

templates/global/values.yaml

@@ -13,6 +13,9 @@ features:
       username: admin
       password: admin123
 
+  cross_region_replication:
+    enabled: false
+
   utapi:
     enabled: false
 

templates/kafka/setup.sh

@@ -39,7 +39,7 @@ if [[ -n "$TOPICS_TO_CREATE" ]]; then
     done
 fi
 
-if [[ "$CREATE_NOTIFICATION_PATHS" == "true" ]]; then
+if [[ "$CREATE_ZOOKEEPER_PATHS" == "true" ]]; then
     if [[ -z "$ZOOKEEPER_ENDPOINT" ]]; then
         echo "[setup] Zookeeper endpoint not set"
         exit 1
@@ -56,6 +56,14 @@ create /bucket-notification/raft-id-dispatcher/provisions
 create /bucket-notification/raft-id-dispatcher/provisions/1
 create /bucket-notification/raft-id-dispatcher/provisions/2
 create /bucket-notification/raft-id-dispatcher/provisions/3
+create /queue-populator
+create /queue-populator/raft-id-dispatcher
+create /queue-populator/raft-id-dispatcher/owners
+create /queue-populator/raft-id-dispatcher/leaders
+create /queue-populator/raft-id-dispatcher/provisions
+create /queue-populator/raft-id-dispatcher/provisions/0
+create /queue-populator/raft-id-dispatcher/provisions/1
+create /queue-populator/raft-id-dispatcher/provisions/2
 quit
 EOF
     echo "[setup] Zookeeper paths created."