fix: Race condition in webhook certificate renewal with cert-manager self-signed issuer without a dedicated CA certificate

docs/deploy/cert-manager.md

@@ -0,0 +1,96 @@
+# Cert Manager Integration
+
+The AWS Load Balancer Controller uses admission webhooks to validate and mutate resources. These webhooks require TLS certificates to operate securely. You can use cert-manager to automatically provision and manage these certificates.
+
+## Upgrade Notes
+
+When upgrading from a previous version, the following scenarios are handled automatically:
+
+1. If you have existing TLS secrets and `keepTLSSecret: true` (default):
+   - Existing secrets are preserved
+   - No new certificates are created
+   - Your existing certificate setup continues to work as before
+
+2. If you're using cert-manager with a custom issuer:
+   - Set `certManager.issuerRef` to keep using your issuer
+   - The new CA hierarchy will not be created
+   - Your existing certificate configuration is preserved
+
+3. If you're using cert-manager without a custom issuer:
+   - A new CA hierarchy will be created
+   - New certificates will be issued using this CA
+   - The transition is handled automatically by cert-manager
+
+## How it Works
+
+When using cert-manager integration, the controller creates a certificate hierarchy that consists of:
+
+1. A self-signed issuer used only to create the root CA certificate
+2. A root CA certificate with a 5-year validity period
+3. A CA issuer that uses the root certificate to sign webhook serving certificates
+4. Webhook serving certificates with 1-year validity that are automatically renewed
+
+This setup prevents race conditions during certificate renewal by:
+- Using a long-lived (5 years) root CA certificate that remains stable
+- Only renewing the serving certificates while keeping the CA constant
+- Letting cert-manager's CA injector handle caBundle updates in webhook configurations
+
+## Configuration
+
+To enable cert-manager integration, set `enableCertManager: true` in your Helm values.
+
+You can customize the certificate configuration through these values:
+
+```yaml
+enableCertManager: true
+
+certManager:
+  # Webhook serving certificate configuration
+  duration: "8760h0m0s"    # 1 year (default)
+  renewBefore: "720h0m0s"  # 30 days (optional)
+  revisionHistoryLimit: 10 # Optional
+
+  # Root CA certificate configuration
+  rootCert:
+    duration: "43800h0m0s" # 5 years (default)
+
+  # Optional: Use your own issuer instead of the auto-generated one
+  # issuerRef:
+  #   name: my-issuer
+  #   kind: ClusterIssuer
+```
+
+### Using Custom Issuers
+
+If you want to use your own cert-manager issuer instead of the auto-generated CA, you can configure it through `certManager.issuerRef`:
+
+```yaml
+certManager:
+  issuerRef:
+    name: my-issuer
+    kind: ClusterIssuer # or Issuer
+```
+
+When a custom issuer is specified:
+- The controller will not create its own CA certificate chain
+- The specified issuer will be used directly to issue webhook serving certificates
+- You are responsible for ensuring the issuer is properly configured and available
+
+### Certificate Renewal
+
+1. Root CA Certificate:
+   - Valid for 5 years by default
+   - Used only for signing webhook certificates
+   - Not renewed automatically to maintain stability
+
+2. Webhook Serving Certificates:
+   - Valid for 1 year by default
+   - Renewed automatically 30 days before expiry
+   - Updates handled seamlessly by cert-manager
+
+### Best Practices
+
+1. Use the default certificate hierarchy unless you have specific requirements
+2. If using a custom issuer, ensure it's highly available and properly configured
+3. Monitor certificate resources for renewal status and potential issues
+4. Keep cert-manager up to date to benefit from the latest improvements

helm/aws-load-balancer-controller/templates/cert-manager.yaml

@@ -0,0 +1,44 @@
+{{- if and .Values.enableCertManager (not .Values.certManager.issuerRef) -}}
+# Create a selfsigned Issuer, in order to create a root CA certificate for
+# signing webhook serving certificates
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ template "aws-load-balancer-controller.namePrefix" . }}-selfsigned-issuer
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "aws-load-balancer-controller.labels" . | nindent 4 }}
+spec:
+  selfSigned: {}
+---
+# Generate a CA Certificate used to sign certificates for the webhook
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ template "aws-load-balancer-controller.namePrefix" . }}-root-cert
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "aws-load-balancer-controller.labels" . | nindent 4 }}
+spec:
+  secretName: {{ template "aws-load-balancer-controller.namePrefix" . }}-root-cert
+  duration: {{ .Values.certManager.rootCert.duration | default "43800h0m0s" | quote }}
+  issuerRef:
+    name: {{ template "aws-load-balancer-controller.namePrefix" . }}-selfsigned-issuer
+  commonName: "ca.webhook.aws-load-balancer-controller"
+  isCA: true
+  subject:
+    organizations:
+      - aws-load-balancer-controller
+---
+# Create an Issuer that uses the above generated CA certificate to issue certs
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ template "aws-load-balancer-controller.namePrefix" . }}-root-issuer
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "aws-load-balancer-controller.labels" . | nindent 4 }}
+spec:
+  ca:
+    secretName: {{ template "aws-load-balancer-controller.namePrefix" . }}-root-cert
+{{- end -}}

helm/aws-load-balancer-controller/templates/webhook.yaml

@@ -12,9 +12,9 @@ metadata:
     {{- include "aws-load-balancer-controller.labels" . | nindent 4 }}
 webhooks:
 - clientConfig:
-    {{ if not $.Values.enableCertManager -}}
+    {{- if not $.Values.enableCertManager }}
     caBundle: {{ $tls.caCert }}
-    {{ end }}
+    {{- end }}
     service:
       name: {{ template "aws-load-balancer-controller.webhookService" . }}
       namespace: {{ $.Release.Namespace }}
@@ -58,9 +58,9 @@ webhooks:
   sideEffects: None
 {{- if .Values.enableServiceMutatorWebhook }}
 - clientConfig:
-    {{ if not $.Values.enableCertManager -}}
+    {{- if not $.Values.enableCertManager }}
     caBundle: {{ $tls.caCert }}
-    {{ end }}
+    {{- end }}
     service:
       name: {{ template "aws-load-balancer-controller.webhookService" . }}
       namespace: {{ $.Release.Namespace }}
@@ -95,9 +95,9 @@ webhooks:
   sideEffects: None
 {{- end }}
 - clientConfig:
-    {{ if not $.Values.enableCertManager -}}
+    {{- if not $.Values.enableCertManager }}
     caBundle: {{ $tls.caCert }}
-    {{ end }}
+    {{- end }}
     service:
       name: {{ template "aws-load-balancer-controller.webhookService" . }}
       namespace: {{ $.Release.Namespace }}
@@ -130,9 +130,9 @@ metadata:
     {{- include "aws-load-balancer-controller.labels" . | nindent 4 }}
 webhooks:
 - clientConfig:
-    {{ if not $.Values.enableCertManager -}}
+    {{- if not $.Values.enableCertManager }}
     caBundle: {{ $tls.caCert }}
-    {{ end }}
+    {{- end }}
     service:
       name: {{ template "aws-load-balancer-controller.webhookService" . }}
       namespace: {{ $.Release.Namespace }}
@@ -159,9 +159,9 @@ webhooks:
     - ingressclassparams
   sideEffects: None
 - clientConfig:
-    {{ if not $.Values.enableCertManager -}}
+    {{- if not $.Values.enableCertManager }}
     caBundle: {{ $tls.caCert }}
-    {{ end }}
+    {{- end }}
     service:
       name: {{ template "aws-load-balancer-controller.webhookService" . }}
       namespace: {{ $.Release.Namespace }}
@@ -183,9 +183,9 @@ webhooks:
   sideEffects: None
 {{- if not $.Values.webhookConfig.disableIngressValidation }}
 - clientConfig:
-    {{ if not $.Values.enableCertManager -}}
+    {{- if not $.Values.enableCertManager }}
     caBundle: {{ $tls.caCert }}
-    {{ end }}
+    {{- end }}
     service:
       name: {{ template "aws-load-balancer-controller.webhookService" . }}
       namespace: {{ $.Release.Namespace }}
@@ -222,6 +222,9 @@ data:
   tls.crt: {{ $tls.clientCert }}
   tls.key: {{ $tls.clientKey }}
 {{- else }}
+{{- $secretName := (include "aws-load-balancer-controller.webhookCertSecret" .) -}}
+{{- $secret := lookup "v1" "Secret" .Release.Namespace $secretName -}}
+{{- if not (and .Values.keepTLSSecret $secret) }}
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
@@ -234,12 +237,16 @@ spec:
   - {{ template "aws-load-balancer-controller.webhookService" . }}.{{ .Release.Namespace }}.svc
   - {{ template "aws-load-balancer-controller.webhookService" . }}.{{ .Release.Namespace }}.svc.{{ .Values.cluster.dnsDomain }}
   issuerRef:
+    {{- if .Values.certManager.issuerRef }}
+    {{- toYaml .Values.certManager.issuerRef | nindent 4 }}
+    {{- else }}
     kind: Issuer
-    name: {{ template "aws-load-balancer-controller.namePrefix" . }}-selfsigned-issuer
+    name: {{ template "aws-load-balancer-controller.namePrefix" . }}-root-issuer
+    {{- end }}
   secretName: {{ template "aws-load-balancer-controller.webhookCertSecret" . }}
   {{- with .Values.certManager -}}
   {{ if .duration }}
-  duration: {{ .duration }}
+  duration: {{ .duration | default "8760h0m0s" | quote }}
   {{- end }}
   {{- if .renewBefore }}
   renewBefore: {{ .renewBefore }}
@@ -248,14 +255,5 @@ spec:
   revisionHistoryLimit: {{ .revisionHistoryLimit }}
   {{- end }}
   {{- end }}
----
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
-  name: {{ template "aws-load-balancer-controller.namePrefix" . }}-selfsigned-issuer
-  namespace: {{ .Release.Namespace }}
-  labels:
-{{ include "aws-load-balancer-controller.labels" . | indent 4 }}
-spec:
-  selfSigned: {}
+{{- end }}
 {{- end }}

helm/aws-load-balancer-controller/values.yaml

@@ -117,10 +117,20 @@ enableCertManager: false
 
 # Overrideable variables when enableCertManager is set to true
 certManager:
-  duration:
-  renewBefore:
+  # Webhook serving certificate configuration
+  duration: "8760h0m0s"  # 1 year
+  renewBefore: "720h0m0s"  # 30 days
   revisionHistoryLimit:
 
+  # Root CA certificate configuration
+  rootCert:
+    duration: "43800h0m0s"  # 5 years
+
+  # Optional: custom issuer reference
+  # issuerRef:
+  #   name: my-issuer
+  #   kind: ClusterIssuer
+
 # The name of the Kubernetes cluster. A non-empty value is required
 clusterName: