feat: extend concurrency limit to all reconcilers

Extend the concurrency control from Stack-only to ALL reconcilers
(Stacks, Modules, and Resources) to truly prevent "big bang" deployments.

Key changes:
- Move concurrency config to core package (internal/core/concurrency.go)
- Apply MaxConcurrentReconciles to all Module reconcilers:
  * Analytics, Auth, Gateway, Ledger, Orchestration
  * Payments, Reconciliation, Search, Stargate
  * Wallets, Webhooks
- Rename env var: STACK_MAX_CONCURRENT → MAX_CONCURRENT_RECONCILES
- Rename Helm param: stackMaxConcurrent → maxConcurrentReconciles
- Update documentation to reflect global scope

This now controls:
- Stack reconciliations (namespace creation, config updates)
- Module reconciliations (Ledger, Payments, etc. deployments)
- Resource reconciliations (Database, Broker management)

Default remains 5 concurrent reconciliations for all resource types.

Author: flemzord

helm/operator/STACK_CONCURRENCY.md

@@ -1,8 +1,8 @@
-# Stack Concurrency Configuration
+# Concurrency Control Configuration
 
 ## Overview
 
-Control the number of Stack reconciliations that run in parallel to prevent cluster overload.
+Control the number of concurrent reconciliations (Stacks, Modules, and other resources) that run in parallel to prevent cluster overload and manage deployment pace.
 
 ## Configuration
 
@@ -12,14 +12,14 @@ Edit your `values.yaml` or use `--set`:
 
 ```yaml
 operator:
-  stackMaxConcurrent: 5  # Max 5 concurrent stack reconciliations
+  maxConcurrentReconciles: 5  # Max 5 concurrent reconciliations (all resources)
 ```
 
 Or with Helm command:
 
 ```bash
 helm install operator ./helm/operator \
-  --set operator.stackMaxConcurrent=5
+  --set operator.maxConcurrentReconciles=5
 ```
 
 ### Default Behavior
@@ -43,7 +43,7 @@ helm install operator ./helm/operator \
 ```yaml
 # values.yaml
 operator:
-  stackMaxConcurrent: 3
+  maxConcurrentReconciles: 3
 ```
 
 ```bash
@@ -55,7 +55,7 @@ helm upgrade operator ./helm/operator -f values.yaml
 ```yaml
 # values-prod.yaml
 operator:
-  stackMaxConcurrent: 10
+  maxConcurrentReconciles: 10
   enableLeaderElection: true
   region: "eu-west-1"
   env: "production"
@@ -69,16 +69,16 @@ helm upgrade operator ./helm/operator -f values-prod.yaml
 
 ```bash
 helm upgrade operator ./helm/operator \
-  --set operator.stackMaxConcurrent=5 \
+  --set operator.maxConcurrentReconciles=5 \
   --set operator.region=us-east-1
 ```
 
 ## How It Works
 
-1. The Helm chart sets the `STACK_MAX_CONCURRENT` environment variable
+1. The Helm chart sets the `MAX_CONCURRENT_RECONCILES` environment variable
 2. The operator reads this value on startup
-3. Stack reconciliations are limited to N concurrent executions
-4. Additional stacks are queued automatically by Kubernetes
+3. All reconciliations (Stacks, Modules like Ledger/Payments, etc.) are limited to N concurrent executions
+4. Additional reconciliations are queued automatically by Kubernetes
 
 ### Behavior
 
@@ -111,12 +111,12 @@ Check if the environment variable is set:
 POD=$(kubectl get pods -n formance-system -l control-plane=formance-controller-manager -o jsonpath='{.items[0].metadata.name}')
 
 # Check environment variables
-kubectl exec -n formance-system $POD -- env | grep STACK_MAX_CONCURRENT
+kubectl exec -n formance-system $POD -- env | grep MAX_CONCURRENT_RECONCILES
 ```
 
 Expected output:
 ```
-STACK_MAX_CONCURRENT=5
+MAX_CONCURRENT_RECONCILES=5
 ```
 
 ## Troubleshooting
@@ -130,7 +130,7 @@ STACK_MAX_CONCURRENT=5
 
 2. **Verify deployment:**
    ```bash
-   kubectl get deployment operator-manager -n formance-system -o yaml | grep -A 2 "STACK_MAX_CONCURRENT"
+   kubectl get deployment operator-manager -n formance-system -o yaml | grep -A 2 "MAX_CONCURRENT_RECONCILES"
    ```
 
 3. **Restart pods to apply changes:**
@@ -176,17 +176,17 @@ kubectl logs -n formance-system -l control-plane=formance-controller-manager -f
 ```yaml
 # values-dev.yaml
 operator:
-  stackMaxConcurrent: 2
+  maxConcurrentReconciles: 2
   env: "dev"
 
 # values-staging.yaml
 operator:
-  stackMaxConcurrent: 5
+  maxConcurrentReconciles: 5
   env: "staging"
 
 # values-prod.yaml
 operator:
-  stackMaxConcurrent: 10
+  maxConcurrentReconciles: 10
   env: "production"
 ```
 
@@ -208,7 +208,7 @@ spec:
     helm:
       values: |
         operator:
-          stackMaxConcurrent: 5
+          maxConcurrentReconciles: 5
           region: "eu-west-1"
           env: "production"
 ```
@@ -217,26 +217,30 @@ spec:
 
 ### Implementation
 
-- **Environment Variable:** `STACK_MAX_CONCURRENT`
-- **Read by:** `internal/resources/stacks/config.go::GetStackConcurrency()`
-- **Applied in:** `internal/resources/stacks/init.go`
+- **Environment Variable:** `MAX_CONCURRENT_RECONCILES`
+- **Read by:** `internal/core/concurrency.go::GetMaxConcurrentReconciles()`
+- **Applied in:** All reconcilers (Stacks, Modules, Resources)
 - **Uses:** Native controller-runtime `MaxConcurrentReconciles`
 
 ### Source Code
 
 ```go
-// internal/resources/stacks/config.go
-func GetStackConcurrency() int {
-    if v := os.Getenv("STACK_MAX_CONCURRENT"); v != "" {
-        if n, err := strconv.Atoi(v); err == nil && n > 0 {
+// internal/core/concurrency.go
+func GetMaxConcurrentReconciles() int {
+    if v := os.Getenv("MAX_CONCURRENT_RECONCILES"); v != "" {
+        if n, err := strconv.Atoi(v); err == nil && n >= 0 {
             return n
         }
     }
-    return 0 // Default: unlimited
+    return 5 // Default: 5 concurrent reconciliations
 }
 ```
 
-## Related Documentation
+## What This Controls
 
-- [How to Limit Concurrent Stacks](../../docs/HOW_TO_LIMIT_CONCURRENT_STACKS.md)
-- [Concurrent Limit Implementation](../../CONCURRENT_LIMIT_IMPLEMENTATION.md)
+This setting limits **all types** of reconciliations:
+- **Stack reconciliations**: Namespace creation, configuration updates
+- **Module reconciliations**: Ledger, Payments, Wallets, Gateway, etc. deployments
+- **Resource reconciliations**: Database, Broker, BrokerTopic management
+
+This prevents "big bang" deployments where all resources are processed simultaneously.

helm/operator/templates/deployment.yaml

@@ -81,10 +81,10 @@ spec:
               port: {{ regexReplaceAll ":" .Values.operator.probeAddr "" | default "8081" }}
             initialDelaySeconds: 5
             periodSeconds: 10
-          {{- if .Values.operator.stackMaxConcurrent }}
+          {{- if .Values.operator.maxConcurrentReconciles }}
           env:
-            - name: STACK_MAX_CONCURRENT
-              value: {{ .Values.operator.stackMaxConcurrent | quote }}
+            - name: MAX_CONCURRENT_RECONCILES
+              value: {{ .Values.operator.maxConcurrentReconciles | quote }}
           {{- end }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}

helm/operator/values.yaml

@@ -49,11 +49,11 @@ operator:
   # Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.
   enableLeaderElection: true
 
-  # Maximum number of concurrent stack reconciliations
+  # Maximum number of concurrent reconciliations (applies to all resources: stacks, modules, etc.)
   # Set to 0 for unlimited
   # Recommended values: 5 for small/medium clusters, 10 for large, 20 for XL
-  # @section -- Stack Concurrency
-  stackMaxConcurrent: 5
+  # @section -- Concurrency Control
+  maxConcurrentReconciles: 5
 
   utils:
     tag: ""

internal/core/concurrency.go

@@ -0,0 +1,18 @@
+package core
+
+import (
+	"os"
+	"strconv"
+)
+
+// GetMaxConcurrentReconciles returns the maximum number of concurrent reconciliations
+// from the MAX_CONCURRENT_RECONCILES environment variable, or a default value of 5
+func GetMaxConcurrentReconciles() int {
+	if v := os.Getenv("MAX_CONCURRENT_RECONCILES"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n >= 0 {
+			return n
+		}
+	}
+	// Default: 5 concurrent reconciliations (good balance for most clusters)
+	return 5
+}

internal/resources/analytics/init.go

@@ -31,6 +31,8 @@ func Reconcile(_ Context, _ *v1beta1.Stack, _ *v1beta1.Analytics, _ string) erro
 
 func init() {
 	Init(
-		WithModuleReconciler(Reconcile),
+		WithModuleReconciler(Reconcile,
+			WithMaxConcurrentReconciles[*v1beta1.Analytics](GetMaxConcurrentReconciles()),
+		),
 	)
 }

internal/resources/auths/init.go

@@ -110,6 +110,7 @@ func Reconcile(ctx Context, stack *v1beta1.Stack, auth *v1beta1.Auth, version st
 func init() {
 	Init(
 		WithModuleReconciler(Reconcile,
+			WithMaxConcurrentReconciles[*v1beta1.Auth](GetMaxConcurrentReconciles()),
 			WithOwn[*v1beta1.Auth](&appsv1.Deployment{}),
 			WithOwn[*v1beta1.Auth](&v1beta1.GatewayHTTPAPI{}),
 			WithOwn[*v1beta1.Auth](&v1beta1.Database{}),

internal/resources/gateways/init.go

@@ -87,6 +87,7 @@ func Reconcile(ctx Context, stack *v1beta1.Stack, gateway *v1beta1.Gateway, vers
 func init() {
 	Init(
 		WithModuleReconciler(Reconcile,
+			WithMaxConcurrentReconciles[*v1beta1.Gateway](GetMaxConcurrentReconciles()),
 			WithOwn[*v1beta1.Gateway](&corev1.ConfigMap{}),
 			WithOwn[*v1beta1.Gateway](&appsv1.Deployment{}),
 			WithOwn[*v1beta1.Gateway](&corev1.Service{}),

internal/resources/ledgers/init.go

@@ -150,6 +150,7 @@ func Reconcile(ctx Context, stack *v1beta1.Stack, ledger *v1beta1.Ledger, versio
 func init() {
 	Init(
 		WithModuleReconciler(Reconcile,
+			WithMaxConcurrentReconciles[*v1beta1.Ledger](GetMaxConcurrentReconciles()),
 			WithOwn[*v1beta1.Ledger](&appsv1.Deployment{}),
 			WithOwn[*v1beta1.Ledger](&batchv1.Job{}),
 			WithOwn[*v1beta1.Ledger](&corev1.Service{}),

internal/resources/orchestrations/init.go

@@ -88,6 +88,7 @@ func Reconcile(ctx Context, stack *v1beta1.Stack, o *v1beta1.Orchestration, vers
 func init() {
 	Init(
 		WithModuleReconciler(Reconcile,
+			WithMaxConcurrentReconciles[*v1beta1.Orchestration](GetMaxConcurrentReconciles()),
 			WithOwn[*v1beta1.Orchestration](&v1beta1.BrokerConsumer{}),
 			WithOwn[*v1beta1.Orchestration](&v1beta1.AuthClient{}),
 			WithOwn[*v1beta1.Orchestration](&appsv1.Deployment{}),

internal/resources/payments/init.go

@@ -126,6 +126,7 @@ func Reconcile(ctx Context, stack *v1beta1.Stack, p *v1beta1.Payments, version s
 func init() {
 	Init(
 		WithModuleReconciler(Reconcile,
+			WithMaxConcurrentReconciles[*v1beta1.Payments](GetMaxConcurrentReconciles()),
 			WithFinalizer[*v1beta1.Payments]("clean-payments", Clean),
 			WithOwn[*v1beta1.Payments](&appsv1.Deployment{}),
 			WithOwn[*v1beta1.Payments](&corev1.Service{}),