diff --git a/charts/kagenti-operator/templates/manager/manager.yaml b/charts/kagenti-operator/templates/manager/manager.yaml index e0ce1f60..d15a92fd 100644 --- a/charts/kagenti-operator/templates/manager/manager.yaml +++ b/charts/kagenti-operator/templates/manager/manager.yaml @@ -133,6 +133,10 @@ spec: - name: KAGENTI_SPIRE_TRUST_DOMAIN value: {{ .Values.signatureVerification.spireTrustDomain | quote }} {{- end }} + {{- if and .Values.signatureVerification .Values.signatureVerification.spireClusterName }} + - name: KAGENTI_SPIRE_CLUSTER_NAME + value: {{ .Values.signatureVerification.spireClusterName | quote }} + {{- end }} {{- if .Values.controllerManager.container.env }} {{- range $key, $value := .Values.controllerManager.container.env }} - name: {{ $key }} diff --git a/charts/kagenti-operator/templates/rbac/role.yaml b/charts/kagenti-operator/templates/rbac/role.yaml index 0db3e12c..a861d1ea 100755 --- a/charts/kagenti-operator/templates/rbac/role.yaml +++ b/charts/kagenti-operator/templates/rbac/role.yaml @@ -251,10 +251,17 @@ rules: - apiGroups: - operator.openshift.io resources: + - spiffecsidrivers + - spireagents + - spireoidcdiscoveryproviders + - spireservers - zerotrustworkloadidentitymanagers verbs: + - create - get - list + - update + - watch - apiGroups: - operator.tekton.dev resources: diff --git a/charts/kagenti-operator/values.yaml b/charts/kagenti-operator/values.yaml index d1709372..4b582d1f 100644 --- a/charts/kagenti-operator/values.yaml +++ b/charts/kagenti-operator/values.yaml @@ -164,6 +164,8 @@ signatureVerification: # Set here only to override auto-discovery. Used by ClientRegistrationReconciler # to build SPIFFE-shaped Keycloak client IDs (spiffe:///ns//sa/). spireTrustDomain: "" + # SPIRE cluster name for the ZTWIM CR. Defaults to "agent-platform" if empty. + spireClusterName: "" # Key within the SPIRE trust bundle ConfigMap. Matches the SPIRE hardened Helm chart default # and the binary flag default. Override to "bundle.crt" only for older ZTWIM deployments. spireTrustBundle: diff --git a/kagenti-operator/cmd/main.go b/kagenti-operator/cmd/main.go index b815d073..0b978561 100644 --- a/kagenti-operator/cmd/main.go +++ b/kagenti-operator/cmd/main.go @@ -743,6 +743,35 @@ func main() { } } + if controller.SpireOperandCRDExists(mgr.GetConfig()) { + if spireTrustDomain == "" { + setupLog.Info("ZTWIM CRDs present but SPIRE trust domain not available; " + + "SPIRE operand controller will not start until trust domain is discoverable") + } else { + spireClusterName := os.Getenv("KAGENTI_SPIRE_CLUSTER_NAME") + if err := mgr.Add(&controller.SpireBootstrapRunnable{ + Client: mgr.GetClient(), + TrustDomain: spireTrustDomain, + ClusterName: spireClusterName, + Log: ctrl.Log.WithName("spire-bootstrap"), + }); err != nil { + setupLog.Error(err, "unable to add SPIRE bootstrap runnable") + os.Exit(1) + } + if err = (&controller.SpireOperandReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("spire-operand-controller"), + TrustDomain: spireTrustDomain, + ClusterName: spireClusterName, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "SpireOperand") + os.Exit(1) + } + setupLog.Info("SPIRE operand controller enabled", "trustDomain", spireTrustDomain) + } + } + // Validation webhooks // For local testing without webhook certificates, set ENABLE_WEBHOOKS=false: // ENABLE_WEBHOOKS=false ./bin/manager --leader-elect=false [other flags...] diff --git a/kagenti-operator/config/rbac/role.yaml b/kagenti-operator/config/rbac/role.yaml index 88e1adb1..ea9af69a 100644 --- a/kagenti-operator/config/rbac/role.yaml +++ b/kagenti-operator/config/rbac/role.yaml @@ -250,10 +250,17 @@ rules: - apiGroups: - operator.openshift.io resources: + - spiffecsidrivers + - spireagents + - spireoidcdiscoveryproviders + - spireservers - zerotrustworkloadidentitymanagers verbs: + - create - get - list + - update + - watch - apiGroups: - operator.tekton.dev resources: diff --git a/kagenti-operator/docs/architecture.md b/kagenti-operator/docs/architecture.md index c86562a2..e6f4f621 100644 --- a/kagenti-operator/docs/architecture.md +++ b/kagenti-operator/docs/architecture.md @@ -75,6 +75,14 @@ The Kagenti Operator is a Kubernetes controller that implements the [Operator Pa - On CR deletion: removes type label, managed-by label and config-hash annotation (causing the workload to lose sidecars) - Coordinates with the AuthBridge mutating webhook (in-process) which injects sidecars at Pod CREATE time +#### SPIRE Operand Controller +- Creates and reconciles 5 SPIRE operand CRs (`operator.openshift.io/v1alpha1`) when ZTWIM CRDs are present on the cluster +- CRD-gated: no feature flag needed — automatically activates on OpenShift 4.19+ where ZTWIM operator is installed +- Manages: ZeroTrustWorkloadIdentityManager (parent), SpiffeCSIDriver, SpireServer, SpireAgent, SpireOIDCDiscoveryProvider (children) +- Drift correction: any manual change to CR spec or deletion is automatically corrected +- Replaces fragile Helm post-install hooks with proper reconciliation loop +- Uses `SpireBootstrapRunnable` to create initial ZTWIM CR at startup, triggering the controller's watch + ### Supporting Components #### Webhooks @@ -108,6 +116,7 @@ graph TB CardController[AgentCard Controller] SyncController[AgentCardSync Controller] RuntimeController[AgentRuntime Controller] + SpireController[SPIRE Operand Controller] CardCR -->|Validates| ValidationWebhook RuntimeCR -->|Validates| ValidationWebhook Deployment -->|CREATE/UPDATE with kagenti.io/type| VAP @@ -115,6 +124,16 @@ graph TB ValidationWebhook -->|Valid CR| CardController end + subgraph "SPIRE Infrastructure" + ZTWIMCR[ZTWIM CR] + SpireCRs[SpireServer + SpireAgent + SpiffeCSIDriver + SpireOIDC] + ZTWIMOperator[ZTWIM Operator] + SpireController -->|Creates/Updates| ZTWIMCR + SpireController -->|Creates/Updates| SpireCRs + ZTWIMOperator -->|Reconciles| ZTWIMCR + ZTWIMOperator -->|Reconciles| SpireCRs + end + subgraph "Config Sources" ClusterCM[Cluster Defaults ConfigMaps] NsCM[Namespace Defaults ConfigMap] @@ -261,6 +280,60 @@ AgentRuntime CR created/updated | `ConfigResolved` | Configuration merged successfully. Reason is `ConfigResolved` when clean, `ConfigWarning` when ambiguity detected (e.g., multiple namespace defaults ConfigMaps). Warnings are surfaced in the condition message and as Kubernetes events. | | `Ready` | Labels and config-hash applied successfully | +### SPIRE Operand Controller + +The SPIRE Operand Controller manages the lifecycle of 5 SPIRE operand CRs on OpenShift clusters where the ZTWIM (Zero Trust Workload Identity Manager) operator is installed. It replaces fragile Helm post-install hooks with a proper reconciliation loop that corrects drift. + +#### CRD Gating + +The controller uses `SpireOperandCRDExists()` to check for the `ZeroTrustWorkloadIdentityManager` CRD via the Kubernetes discovery API (3 retries). No feature flag is needed — the controller activates automatically when CRDs are present (OpenShift 4.19+). + +#### Bootstrap Flow + +`SpireBootstrapRunnable` is a one-shot `manager.Runnable` that creates the initial ZTWIM CR at startup if absent. This triggers the controller's watch, which then creates the 4 child CRs. + +#### Reconciliation Flow + +``` +1. Get ZTWIM CR "cluster" — if NotFound, return (bootstrap pending) +2. ensureUnstructuredCR(ZTWIM) — CreateOrUpdate with desired spec: + - trustDomain (auto-discovered), clusterName="agent-platform", bundleConfigMap="spire-bundle" +3. ensureChildren() — CreateOrUpdate for each of 4 children: + a. SpiffeCSIDriver: agentSocketPath, pluginName + b. SpireServer: caSubject, persistence, datastore, jwtIssuer + c. SpireAgent: nodeAttestor, workloadAttestors + d. SpireOIDCDiscoveryProvider: csiDriverName, jwtIssuer +4. Record events on create/update, return success +``` + +Children inherit `trustDomain` and `clusterName` from the parent ZTWIM CR — these fields are NOT set on child CRs (OCP 4.19 CRD constraint). + +#### Managed CRs + +All CRs are `operator.openshift.io/v1alpha1`, cluster-scoped, name `"cluster"`: + +| CR | Key Spec Fields | Role | +|---|---|---| +| ZeroTrustWorkloadIdentityManager | trustDomain, clusterName, bundleConfigMap | Parent — created first | +| SpiffeCSIDriver | agentSocketPath, pluginName | CSI volume plugin for SVID mounting | +| SpireServer | caSubject, persistence, datastore, jwtIssuer | SPIRE server configuration | +| SpireAgent | nodeAttestor, workloadAttestors | Node-level SPIRE agent | +| SpireOIDCDiscoveryProvider | csiDriverName, jwtIssuer | OIDC endpoint for JWT-SVID | + +#### Watches + +| Resource | Scope | Purpose | +|----------|-------|---------| +| ZeroTrustWorkloadIdentityManager | Cluster | Primary resource — reconcile on create/update/delete | +| SpiffeCSIDriver | Cluster | Secondary — map to ZTWIM reconcile for drift correction | +| SpireServer | Cluster | Secondary — map to ZTWIM reconcile for drift correction | +| SpireAgent | Cluster | Secondary — map to ZTWIM reconcile for drift correction | +| SpireOIDCDiscoveryProvider | Cluster | Secondary — map to ZTWIM reconcile for drift correction | + +#### Drift Correction + +Uses `controllerutil.CreateOrUpdate` — if a CR exists but spec differs from desired state, it is updated. If a CR is deleted, it is recreated on next reconcile (triggered by the child watch). All CRs are labeled `app.kubernetes.io/managed-by: kagenti-operator`. + ### NetworkPolicy Controller The NetworkPolicy Controller enforces network isolation based on signature verification. @@ -356,6 +429,18 @@ Source: `internal/controller/agentcard_networkpolicy_controller.go` | `networking.k8s.io` | `networkpolicies` | get, list, watch, create, update, patch, delete | Create permissive/restrictive NetworkPolicies based on signature verification | | `""` (core) | `pods` | get, list, watch, update, patch | Resolve pod selectors from workload pod template labels | +#### SPIRE Operand Controller Permissions + +Source: `internal/controller/spire_operand_controller.go` + +| API Group | Resources | Verbs | Purpose | +|-----------|-----------|-------|---------| +| `operator.openshift.io` | `zerotrustworkloadidentitymanagers` | create, get, list, update, watch | Create and reconcile ZTWIM parent CR | +| `operator.openshift.io` | `spiffecsidrivers` | create, get, list, update, watch | Create and reconcile SpiffeCSIDriver CR | +| `operator.openshift.io` | `spireservers` | create, get, list, update, watch | Create and reconcile SpireServer CR | +| `operator.openshift.io` | `spireagents` | create, get, list, update, watch | Create and reconcile SpireAgent CR | +| `operator.openshift.io` | `spireoidcdiscoveryproviders` | create, get, list, update, watch | Create and reconcile SpireOIDCDiscoveryProvider CR | + #### Cross-Namespace Considerations - The operator uses a **ClusterRole** and **ClusterRoleBinding** in cluster-wide mode, allowing it to reconcile resources across all namespaces diff --git a/kagenti-operator/docs/operator.md b/kagenti-operator/docs/operator.md index 2b6bf90c..475b452f 100644 --- a/kagenti-operator/docs/operator.md +++ b/kagenti-operator/docs/operator.md @@ -61,6 +61,9 @@ Reconciles `AgentCard` CRs by resolving the backing workload via `spec.targetRef ### AgentCard Sync Controller Watches Deployments and StatefulSets labeled with `kagenti.io/type=agent` and one or more `protocol.kagenti.io/` labels (e.g., `protocol.kagenti.io/a2a`). Automatically creates AgentCard resources with `targetRef` pointing to the discovered workloads. Sets owner references for garbage collection. +### SPIRE Operand Controller +Creates and reconciles 5 SPIRE operand CRs (`operator.openshift.io/v1alpha1`) when ZTWIM CRDs are present on the cluster. CRD-gated — no feature flag needed, automatically activates on OpenShift 4.19+. Manages ZeroTrustWorkloadIdentityManager (parent), SpiffeCSIDriver, SpireServer, SpireAgent, and SpireOIDCDiscoveryProvider (children). Corrects drift on any manual change or deletion. Uses a bootstrap runnable to create the initial ZTWIM CR at startup. + ### NetworkPolicy Controller Watches AgentCard resources when `--enforce-network-policies` is enabled. Creates permissive NetworkPolicies for agents with verified signatures and restrictive NetworkPolicies for unverified agents. When identity binding is configured, both signature and binding must pass for permissive access. diff --git a/kagenti-operator/internal/controller/spire_operand_controller.go b/kagenti-operator/internal/controller/spire_operand_controller.go new file mode 100644 index 00000000..1201b501 --- /dev/null +++ b/kagenti-operator/internal/controller/spire_operand_controller.go @@ -0,0 +1,403 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "time" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + kd "github.com/kagenti/operator/internal/discovery" +) + +const ( + spireOperandName = "cluster" +) + +var ( + spireOperandLogger = ctrl.Log.WithName("controller").WithName("SpireOperand") + + spiffeCSIDriverGVK = schema.GroupVersionKind{ + Group: "operator.openshift.io", Version: "v1alpha1", Kind: "SpiffeCSIDriver", + } + spireServerGVK = schema.GroupVersionKind{ + Group: "operator.openshift.io", Version: "v1alpha1", Kind: "SpireServer", + } + spireAgentGVK = schema.GroupVersionKind{ + Group: "operator.openshift.io", Version: "v1alpha1", Kind: "SpireAgent", + } + spireOIDCProviderGVK = schema.GroupVersionKind{ + Group: "operator.openshift.io", Version: "v1alpha1", Kind: "SpireOIDCDiscoveryProvider", + } +) + +// SpireOperandReconciler watches the ZTWIM CR and ensures all 5 SPIRE operand +// CRs exist with the desired spec, correcting drift on each reconcile. +type SpireOperandReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + TrustDomain string + ClusterName string +} + +// +kubebuilder:rbac:groups=operator.openshift.io,resources=zerotrustworkloadidentitymanagers,verbs=create;get;list;update;watch +// +kubebuilder:rbac:groups=operator.openshift.io,resources=spiffecsidrivers,verbs=create;get;list;update;watch +// +kubebuilder:rbac:groups=operator.openshift.io,resources=spireservers,verbs=create;get;list;update;watch +// +kubebuilder:rbac:groups=operator.openshift.io,resources=spireagents,verbs=create;get;list;update;watch +// +kubebuilder:rbac:groups=operator.openshift.io,resources=spireoidcdiscoveryproviders,verbs=create;get;list;update;watch + +func (r *SpireOperandReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + logger.V(1).Info("Reconciling SPIRE operand", "name", req.Name) + + existing := &unstructured.Unstructured{} + existing.SetGroupVersionKind(kd.ZTWIMGVK) + if err := r.Get(ctx, types.NamespacedName{Name: spireOperandName}, existing); err != nil { + if errors.IsNotFound(err) { + logger.V(1).Info("ZTWIM CR not found, bootstrap pending") + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + // trustDomain and clusterName are immutable on the ZTWIM CR — once set + // by the initial creator (bootstrap or Helm), they cannot be changed. + // Read existing values and use them for spec builders so we never + // trigger a rejected update. Fall back to configured values only for + // new CRs. + effectiveTD := r.TrustDomain + if td, found, _ := unstructured.NestedString(existing.Object, "spec", "trustDomain"); found && td != "" { + effectiveTD = td + } + effectiveCN := r.ClusterName + if cn, found, _ := unstructured.NestedString(existing.Object, "spec", "clusterName"); found && cn != "" { + effectiveCN = cn + } + + if err := r.ensureUnstructuredCR(ctx, kd.ZTWIMGVK, spireOperandName, r.ztwimSpec(effectiveTD, effectiveCN)); err != nil { + return ctrl.Result{}, fmt.Errorf("ensuring ZTWIM CR: %w", err) + } + + if err := r.ensureChildren(ctx, effectiveTD); err != nil { + return ctrl.Result{}, fmt.Errorf("ensuring SPIRE children: %w", err) + } + + logger.Info("SPIRE operand reconciliation complete") + return ctrl.Result{}, nil +} + +func (r *SpireOperandReconciler) ensureChildren(ctx context.Context, trustDomain string) error { + children := []struct { + gvk schema.GroupVersionKind + spec map[string]interface{} + }{ + {spiffeCSIDriverGVK, r.spiffeCSIDriverSpec()}, + {spireServerGVK, r.spireServerSpec(trustDomain)}, + {spireAgentGVK, r.spireAgentSpec()}, + {spireOIDCProviderGVK, r.spireOIDCProviderSpec(trustDomain)}, + } + for _, child := range children { + if err := r.ensureUnstructuredCR(ctx, child.gvk, spireOperandName, child.spec); err != nil { + return fmt.Errorf("ensuring %s: %w", child.gvk.Kind, err) + } + } + return nil +} + +func (r *SpireOperandReconciler) ensureUnstructuredCR( + ctx context.Context, gvk schema.GroupVersionKind, + name string, spec map[string]interface{}, +) error { + logger := log.FromContext(ctx) + + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + obj.SetName(name) + + result, err := controllerutil.CreateOrUpdate(ctx, r.Client, obj, func() error { + lbls := obj.GetLabels() + if lbls == nil { + lbls = make(map[string]string) + } + lbls[LabelManagedBy] = LabelManagedByValue + obj.SetLabels(lbls) + + // Merge our fields into existing spec rather than replacing it. + // The ZTWIM operator adds default fields (e.g. disableMigration) + // that we don't manage — replacing the whole spec would cause an + // update loop as each side overwrites the other's fields. + existingSpec, _, _ := unstructured.NestedMap(obj.Object, "spec") + if existingSpec == nil { + existingSpec = make(map[string]interface{}) + } + mergeNestedMap(existingSpec, spec) + if err := unstructured.SetNestedField(obj.Object, existingSpec, "spec"); err != nil { + return fmt.Errorf("setting spec on %s: %w", gvk.Kind, err) + } + return nil + }) + if err != nil { + return err + } + + switch result { + case controllerutil.OperationResultCreated: + logger.Info("Created SPIRE operand CR", "kind", gvk.Kind, "name", name) + if r.Recorder != nil { + r.Recorder.Event(obj, "Normal", "Created", + fmt.Sprintf("Created %s/%s", gvk.Kind, name)) + } + case controllerutil.OperationResultUpdated: + logger.Info("Updated SPIRE operand CR (drift corrected)", "kind", gvk.Kind, "name", name) + if r.Recorder != nil { + r.Recorder.Event(obj, "Normal", "Updated", + fmt.Sprintf("Updated %s/%s (drift corrected)", gvk.Kind, name)) + } + } + + return nil +} + +// mergeNestedMap recursively merges src into dst. For nested maps, it recurses. +// For all other types, src values overwrite dst values. Keys in dst not present +// in src are preserved — this prevents overwriting fields set by the ZTWIM operator. +func mergeNestedMap(dst, src map[string]interface{}) { + for k, v := range src { + if srcMap, ok := v.(map[string]interface{}); ok { + if dstMap, ok := dst[k].(map[string]interface{}); ok { + mergeNestedMap(dstMap, srcMap) + continue + } + } + dst[k] = v + } +} + +// --- Spec builders --- + +func (r *SpireOperandReconciler) ztwimSpec(trustDomain, clusterName string) map[string]interface{} { + if clusterName == "" { + clusterName = "agent-platform" + } + return map[string]interface{}{ + "trustDomain": trustDomain, + "clusterName": clusterName, + "bundleConfigMap": "spire-bundle", + } +} + +func (r *SpireOperandReconciler) spiffeCSIDriverSpec() map[string]interface{} { + return map[string]interface{}{ + "agentSocketPath": "/run/spire/agent-sockets", + "pluginName": "csi.spiffe.io", + } +} + +func (r *SpireOperandReconciler) spireServerSpec(trustDomain string) map[string]interface{} { + return map[string]interface{}{ + "caSubject": map[string]interface{}{ + "commonName": trustDomain, + "country": "US", + "organization": "RH", + }, + "persistence": map[string]interface{}{ + "size": "5Gi", + "accessMode": "ReadWriteOnce", + }, + "datastore": map[string]interface{}{ + "databaseType": "sqlite3", + "connectionString": "/run/spire/data/datastore.sqlite3", + "maxOpenConns": int64(100), + "maxIdleConns": int64(2), + "connMaxLifetime": int64(3600), + }, + "jwtIssuer": fmt.Sprintf("https://oidc-discovery-provider.%s", trustDomain), + } +} + +func (r *SpireOperandReconciler) spireAgentSpec() map[string]interface{} { + return map[string]interface{}{ + "nodeAttestor": map[string]interface{}{ + "k8sPSATEnabled": "true", + }, + "workloadAttestors": map[string]interface{}{ + "k8sEnabled": "true", + "workloadAttestorsVerification": map[string]interface{}{ + "type": "auto", + }, + }, + } +} + +func (r *SpireOperandReconciler) spireOIDCProviderSpec(trustDomain string) map[string]interface{} { + return map[string]interface{}{ + "csiDriverName": "csi.spiffe.io", + "jwtIssuer": fmt.Sprintf("https://oidc-discovery-provider.%s", trustDomain), + } +} + +// --- Controller setup --- + +func mapChildToZTWIM(_ context.Context, _ client.Object) []reconcile.Request { + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{Name: spireOperandName}}, + } +} + +func (r *SpireOperandReconciler) SetupWithManager(mgr ctrl.Manager) error { + _, err := mgr.GetRESTMapper().RESTMapping( + kd.ZTWIMGVK.GroupKind(), kd.ZTWIMGVK.Version, + ) + if err != nil { + if meta.IsNoMatchError(err) { + log.Log.Info("ZTWIM CRD not registered, SPIRE operand controller will not start") + return nil + } + return fmt.Errorf("checking ZTWIM CRD: %w", err) + } + + ztwim := &unstructured.Unstructured{} + ztwim.SetGroupVersionKind(kd.ZTWIMGVK) + + bld := ctrl.NewControllerManagedBy(mgr). + For(ztwim). + Named("spire-operand") + + for _, gvk := range []schema.GroupVersionKind{ + spiffeCSIDriverGVK, spireServerGVK, spireAgentGVK, spireOIDCProviderGVK, + } { + child := &unstructured.Unstructured{} + child.SetGroupVersionKind(gvk) + bld.Watches(child, handler.EnqueueRequestsFromMapFunc(mapChildToZTWIM)) + } + + return bld.Complete(r) +} + +// --- CRD existence check --- + +func SpireOperandCRDExists(cfg *rest.Config) bool { + dc, err := discovery.NewDiscoveryClientForConfig(cfg) + if err != nil { + spireOperandLogger.Error(err, "Failed to create discovery client for SPIRE operand check") + return false + } + + for attempt := range 3 { + if attempt > 0 { + delay := time.Duration(attempt) * time.Second + spireOperandLogger.Info("Retrying ZTWIM CRD discovery", "attempt", attempt+1, "delay", delay) + time.Sleep(delay) + } + + resources, err := dc.ServerResourcesForGroupVersion("operator.openshift.io/v1alpha1") + if err != nil { + spireOperandLogger.Info("ZTWIM CRD not found", "attempt", attempt+1, "error", err) + continue + } + + for _, r := range resources.APIResources { + if r.Kind == "ZeroTrustWorkloadIdentityManager" { + spireOperandLogger.Info("ZTWIM CRD detected: will manage SPIRE operand CRs") + return true + } + } + + spireOperandLogger.Info("operator.openshift.io/v1alpha1 group exists but ZTWIM kind not found") + return false + } + + spireOperandLogger.Info("ZTWIM CRD not found after retries: SPIRE operand controller will not start") + return false +} + +// --- Bootstrap runnable --- + +// SpireBootstrapRunnable creates the ZTWIM CR at startup to trigger the controller's watch. +type SpireBootstrapRunnable struct { + Client client.Client + TrustDomain string + ClusterName string + Log logr.Logger +} + +func (b *SpireBootstrapRunnable) Start(ctx context.Context) error { + b.Log.Info("SPIRE bootstrap: ensuring ZTWIM CR exists") + + existing := &unstructured.Unstructured{} + existing.SetGroupVersionKind(kd.ZTWIMGVK) + if err := b.Client.Get(ctx, types.NamespacedName{Name: spireOperandName}, existing); err == nil { + b.Log.Info("SPIRE bootstrap: ZTWIM CR already exists, skipping creation") + return nil + } else if !errors.IsNotFound(err) { + return fmt.Errorf("checking ZTWIM CR existence: %w", err) + } + + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(kd.ZTWIMGVK) + obj.SetName(spireOperandName) + obj.SetLabels(map[string]string{ + LabelManagedBy: LabelManagedByValue, + }) + + clusterName := b.ClusterName + if clusterName == "" { + clusterName = "agent-platform" + } + spec := map[string]interface{}{ + "trustDomain": b.TrustDomain, + "clusterName": clusterName, + "bundleConfigMap": "spire-bundle", + } + if err := unstructured.SetNestedField(obj.Object, spec, "spec"); err != nil { + return fmt.Errorf("setting ZTWIM spec: %w", err) + } + + if err := b.Client.Create(ctx, obj); err != nil { + if errors.IsAlreadyExists(err) { + b.Log.Info("SPIRE bootstrap: ZTWIM CR created concurrently, skipping") + return nil + } + return fmt.Errorf("creating ZTWIM CR: %w", err) + } + + b.Log.Info("SPIRE bootstrap: created ZTWIM CR", "trustDomain", b.TrustDomain) + return nil +} + +// NeedLeaderElection implements manager.LeaderElectionRunnable. +func (b *SpireBootstrapRunnable) NeedLeaderElection() bool { + return true +} diff --git a/kagenti-operator/internal/controller/spire_operand_controller_test.go b/kagenti-operator/internal/controller/spire_operand_controller_test.go new file mode 100644 index 00000000..fc8cd36c --- /dev/null +++ b/kagenti-operator/internal/controller/spire_operand_controller_test.go @@ -0,0 +1,396 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + kd "github.com/kagenti/operator/internal/discovery" +) + +const testTrustDomain = "example.test" + +func newSpireTestScheme() *runtime.Scheme { + s := runtime.NewScheme() + return s +} + +func newSpireReconciler(objs ...runtime.Object) (*SpireOperandReconciler, *fake.ClientBuilder) { + s := newSpireTestScheme() + builder := fake.NewClientBuilder().WithScheme(s) + if len(objs) > 0 { + clientObjs := make([]runtime.Object, len(objs)) + copy(clientObjs, objs) + } + return &SpireOperandReconciler{ + Scheme: s, + TrustDomain: testTrustDomain, + }, builder +} + +func newZTWIM(trustDomain string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(kd.ZTWIMGVK) + obj.SetName(spireOperandName) + obj.SetLabels(map[string]string{ + LabelManagedBy: LabelManagedByValue, + }) + _ = unstructured.SetNestedField(obj.Object, map[string]interface{}{ + "trustDomain": trustDomain, + "clusterName": "agent-platform", + "bundleConfigMap": "spire-bundle", + }, "spec") + return obj +} + +func newChildCR(gvk schema.GroupVersionKind, spec map[string]interface{}) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + obj.SetName(spireOperandName) + obj.SetLabels(map[string]string{ + LabelManagedBy: LabelManagedByValue, + }) + if spec != nil { + _ = unstructured.SetNestedField(obj.Object, spec, "spec") + } + return obj +} + +func spireReconcileRequest() reconcile.Request { + return reconcile.Request{ + NamespacedName: types.NamespacedName{Name: spireOperandName}, + } +} + +var _ = Describe("SPIRE Operand Controller", func() { + ctx := context.Background() + + Context("When ZTWIM CR does not exist", func() { + It("should return without error (bootstrap pending)", func() { + r, builder := newSpireReconciler() + cl := builder.Build() + r.Client = cl + + result, err := r.Reconcile(ctx, spireReconcileRequest()) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) + }) + }) + + Context("When ZTWIM CR exists", func() { + It("should ensure ZTWIM spec matches desired state", func() { + ztwim := newZTWIM(testTrustDomain) + r, builder := newSpireReconciler() + cl := builder.WithObjects(ztwim).Build() + r.Client = cl + + result, err := r.Reconcile(ctx, spireReconcileRequest()) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) + + got := &unstructured.Unstructured{} + got.SetGroupVersionKind(kd.ZTWIMGVK) + Expect(cl.Get(ctx, types.NamespacedName{Name: spireOperandName}, got)).To(Succeed()) + + td, _, _ := unstructured.NestedString(got.Object, "spec", "trustDomain") + Expect(td).To(Equal(testTrustDomain)) + + cn, _, _ := unstructured.NestedString(got.Object, "spec", "clusterName") + Expect(cn).To(Equal("agent-platform")) + + Expect(got.GetLabels()[LabelManagedBy]).To(Equal(LabelManagedByValue)) + }) + + It("should create all 4 child CRs", func() { + ztwim := newZTWIM(testTrustDomain) + r, builder := newSpireReconciler() + cl := builder.WithObjects(ztwim).Build() + r.Client = cl + + result, err := r.Reconcile(ctx, spireReconcileRequest()) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) + + for _, gvk := range []schema.GroupVersionKind{ + spiffeCSIDriverGVK, spireServerGVK, spireAgentGVK, spireOIDCProviderGVK, + } { + child := &unstructured.Unstructured{} + child.SetGroupVersionKind(gvk) + Expect(cl.Get(ctx, types.NamespacedName{Name: spireOperandName}, child)).To(Succeed(), + "expected %s CR to exist", gvk.Kind) + Expect(child.GetLabels()[LabelManagedBy]).To(Equal(LabelManagedByValue)) + } + }) + + It("should set correct SpiffeCSIDriver spec", func() { + ztwim := newZTWIM(testTrustDomain) + r, builder := newSpireReconciler() + cl := builder.WithObjects(ztwim).Build() + r.Client = cl + + _, err := r.Reconcile(ctx, spireReconcileRequest()) + Expect(err).NotTo(HaveOccurred()) + + child := &unstructured.Unstructured{} + child.SetGroupVersionKind(spiffeCSIDriverGVK) + Expect(cl.Get(ctx, types.NamespacedName{Name: spireOperandName}, child)).To(Succeed()) + + path, _, _ := unstructured.NestedString(child.Object, "spec", "agentSocketPath") + Expect(path).To(Equal("/run/spire/agent-sockets")) + + plugin, _, _ := unstructured.NestedString(child.Object, "spec", "pluginName") + Expect(plugin).To(Equal("csi.spiffe.io")) + }) + + It("should set correct SpireServer spec", func() { + ztwim := newZTWIM(testTrustDomain) + r, builder := newSpireReconciler() + cl := builder.WithObjects(ztwim).Build() + r.Client = cl + + _, err := r.Reconcile(ctx, spireReconcileRequest()) + Expect(err).NotTo(HaveOccurred()) + + child := &unstructured.Unstructured{} + child.SetGroupVersionKind(spireServerGVK) + Expect(cl.Get(ctx, types.NamespacedName{Name: spireOperandName}, child)).To(Succeed()) + + cn, _, _ := unstructured.NestedString(child.Object, "spec", "caSubject", "commonName") + Expect(cn).To(Equal(testTrustDomain)) + + dbType, _, _ := unstructured.NestedString(child.Object, "spec", "datastore", "databaseType") + Expect(dbType).To(Equal("sqlite3")) + + issuer, _, _ := unstructured.NestedString(child.Object, "spec", "jwtIssuer") + Expect(issuer).To(Equal("https://oidc-discovery-provider." + testTrustDomain)) + }) + + It("should set correct SpireAgent spec (no trustDomain/clusterName)", func() { + ztwim := newZTWIM(testTrustDomain) + r, builder := newSpireReconciler() + cl := builder.WithObjects(ztwim).Build() + r.Client = cl + + _, err := r.Reconcile(ctx, spireReconcileRequest()) + Expect(err).NotTo(HaveOccurred()) + + child := &unstructured.Unstructured{} + child.SetGroupVersionKind(spireAgentGVK) + Expect(cl.Get(ctx, types.NamespacedName{Name: spireOperandName}, child)).To(Succeed()) + + psat, _, _ := unstructured.NestedString(child.Object, "spec", "nodeAttestor", "k8sPSATEnabled") + Expect(psat).To(Equal("true")) + + // Children must NOT have trustDomain or clusterName + _, found, _ := unstructured.NestedString(child.Object, "spec", "trustDomain") + Expect(found).To(BeFalse()) + _, found, _ = unstructured.NestedString(child.Object, "spec", "clusterName") + Expect(found).To(BeFalse()) + }) + + It("should set correct SpireOIDCDiscoveryProvider spec", func() { + ztwim := newZTWIM(testTrustDomain) + r, builder := newSpireReconciler() + cl := builder.WithObjects(ztwim).Build() + r.Client = cl + + _, err := r.Reconcile(ctx, spireReconcileRequest()) + Expect(err).NotTo(HaveOccurred()) + + child := &unstructured.Unstructured{} + child.SetGroupVersionKind(spireOIDCProviderGVK) + Expect(cl.Get(ctx, types.NamespacedName{Name: spireOperandName}, child)).To(Succeed()) + + driver, _, _ := unstructured.NestedString(child.Object, "spec", "csiDriverName") + Expect(driver).To(Equal("csi.spiffe.io")) + + issuer, _, _ := unstructured.NestedString(child.Object, "spec", "jwtIssuer") + Expect(issuer).To(Equal("https://oidc-discovery-provider." + testTrustDomain)) + }) + }) + + Context("Drift correction", func() { + It("should preserve immutable ZTWIM trustDomain", func() { + existingTD := "existing-domain.org" + ztwim := newZTWIM(existingTD) + r, builder := newSpireReconciler() + cl := builder.WithObjects(ztwim).Build() + r.Client = cl + + _, err := r.Reconcile(ctx, spireReconcileRequest()) + Expect(err).NotTo(HaveOccurred()) + + got := &unstructured.Unstructured{} + got.SetGroupVersionKind(kd.ZTWIMGVK) + Expect(cl.Get(ctx, types.NamespacedName{Name: spireOperandName}, got)).To(Succeed()) + + td, _, _ := unstructured.NestedString(got.Object, "spec", "trustDomain") + Expect(td).To(Equal(existingTD), "trustDomain is immutable — must preserve existing value") + }) + + It("should correct drifted child spec", func() { + ztwim := newZTWIM(testTrustDomain) + driftedCSI := newChildCR(spiffeCSIDriverGVK, map[string]interface{}{ + "agentSocketPath": "/wrong/path", + "pluginName": "wrong.plugin", + }) + r, builder := newSpireReconciler() + cl := builder.WithObjects(ztwim, driftedCSI).Build() + r.Client = cl + + _, err := r.Reconcile(ctx, spireReconcileRequest()) + Expect(err).NotTo(HaveOccurred()) + + got := &unstructured.Unstructured{} + got.SetGroupVersionKind(spiffeCSIDriverGVK) + Expect(cl.Get(ctx, types.NamespacedName{Name: spireOperandName}, got)).To(Succeed()) + + path, _, _ := unstructured.NestedString(got.Object, "spec", "agentSocketPath") + Expect(path).To(Equal("/run/spire/agent-sockets")) + + plugin, _, _ := unstructured.NestedString(got.Object, "spec", "pluginName") + Expect(plugin).To(Equal("csi.spiffe.io")) + }) + }) + + Context("Idempotency", func() { + It("should produce same result on double reconcile", func() { + ztwim := newZTWIM(testTrustDomain) + r, builder := newSpireReconciler() + cl := builder.WithObjects(ztwim).Build() + r.Client = cl + + result1, err := r.Reconcile(ctx, spireReconcileRequest()) + Expect(err).NotTo(HaveOccurred()) + + result2, err := r.Reconcile(ctx, spireReconcileRequest()) + Expect(err).NotTo(HaveOccurred()) + Expect(result2).To(Equal(result1)) + + // All 5 CRs still exist + for _, gvk := range []schema.GroupVersionKind{ + kd.ZTWIMGVK, spiffeCSIDriverGVK, spireServerGVK, spireAgentGVK, spireOIDCProviderGVK, + } { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + Expect(cl.Get(ctx, types.NamespacedName{Name: spireOperandName}, obj)).To(Succeed()) + } + }) + }) + + Context("Bootstrap Runnable", func() { + It("should create ZTWIM when absent", func() { + s := newSpireTestScheme() + cl := fake.NewClientBuilder().WithScheme(s).Build() + b := &SpireBootstrapRunnable{ + Client: cl, + TrustDomain: testTrustDomain, + Log: ctrl.Log.WithName("test-bootstrap"), + } + + err := b.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + + got := &unstructured.Unstructured{} + got.SetGroupVersionKind(kd.ZTWIMGVK) + Expect(cl.Get(ctx, types.NamespacedName{Name: spireOperandName}, got)).To(Succeed()) + + td, _, _ := unstructured.NestedString(got.Object, "spec", "trustDomain") + Expect(td).To(Equal(testTrustDomain)) + Expect(got.GetLabels()[LabelManagedBy]).To(Equal(LabelManagedByValue)) + }) + + It("should skip when ZTWIM already exists", func() { + ztwim := newZTWIM(testTrustDomain) + s := newSpireTestScheme() + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(ztwim).Build() + b := &SpireBootstrapRunnable{ + Client: cl, + TrustDomain: testTrustDomain, + Log: ctrl.Log.WithName("test-bootstrap"), + } + + err := b.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should implement NeedLeaderElection", func() { + b := &SpireBootstrapRunnable{} + Expect(b.NeedLeaderElection()).To(BeTrue()) + }) + }) + + Context("Spec builders", func() { + It("should not include trustDomain in child specs", func() { + r := &SpireOperandReconciler{TrustDomain: testTrustDomain} + + for name, spec := range map[string]map[string]interface{}{ + "SpiffeCSIDriver": r.spiffeCSIDriverSpec(), + "SpireAgent": r.spireAgentSpec(), + "SpireOIDCDiscoveryProvider": r.spireOIDCProviderSpec(testTrustDomain), + } { + _, hasTD := spec["trustDomain"] + Expect(hasTD).To(BeFalse(), "%s should not have trustDomain", name) + _, hasCN := spec["clusterName"] + Expect(hasCN).To(BeFalse(), "%s should not have clusterName", name) + } + }) + + It("should include trustDomain only in ZTWIM spec", func() { + r := &SpireOperandReconciler{TrustDomain: testTrustDomain} + spec := r.ztwimSpec(testTrustDomain, "") + Expect(spec["trustDomain"]).To(Equal(testTrustDomain)) + Expect(spec["clusterName"]).To(Equal("agent-platform")) + Expect(spec["bundleConfigMap"]).To(Equal("spire-bundle")) + }) + + It("should use custom clusterName when provided", func() { + r := &SpireOperandReconciler{TrustDomain: testTrustDomain, ClusterName: "custom-cluster"} + spec := r.ztwimSpec(testTrustDomain, "custom-cluster") + Expect(spec["clusterName"]).To(Equal("custom-cluster")) + }) + + It("should preserve existing trustDomain on reconcile", func() { + existingTD := "existing-domain.org" + ztwim := newZTWIM(existingTD) + r, builder := newSpireReconciler() + r.TrustDomain = "different-discovered-domain.com" + cl := builder.WithObjects(ztwim).Build() + r.Client = cl + + _, err := r.Reconcile(ctx, spireReconcileRequest()) + Expect(err).NotTo(HaveOccurred()) + + got := &unstructured.Unstructured{} + got.SetGroupVersionKind(kd.ZTWIMGVK) + Expect(cl.Get(ctx, types.NamespacedName{Name: spireOperandName}, got)).To(Succeed()) + + td, _, _ := unstructured.NestedString(got.Object, "spec", "trustDomain") + Expect(td).To(Equal(existingTD), "should preserve existing trustDomain, not overwrite with discovered one") + }) + }) +}) diff --git a/kagenti-operator/internal/discovery/discovery.go b/kagenti-operator/internal/discovery/discovery.go index 25a04b32..8be04fc3 100644 --- a/kagenti-operator/internal/discovery/discovery.go +++ b/kagenti-operator/internal/discovery/discovery.go @@ -24,7 +24,8 @@ var ( Version: "v1", Kind: "Route", } - ztwimGVK = schema.GroupVersionKind{ + // ZTWIMGVK is the GroupVersionKind for the ZeroTrustWorkloadIdentityManager CR. + ZTWIMGVK = schema.GroupVersionKind{ Group: "operator.openshift.io", Version: "v1alpha1", Kind: "ZeroTrustWorkloadIdentityManager", @@ -92,7 +93,7 @@ func DiscoverSPIRETrustDomain(ctx context.Context, c client.Reader, spireNamespa func discoverFromZTWIM(ctx context.Context, c client.Reader) (string, error) { ztwim := &unstructured.Unstructured{} - ztwim.SetGroupVersionKind(ztwimGVK) + ztwim.SetGroupVersionKind(ZTWIMGVK) if err := c.Get(ctx, types.NamespacedName{Name: "cluster"}, ztwim); err != nil { return "", err } diff --git a/kagenti-operator/test/e2e/spire_operand_test.go b/kagenti-operator/test/e2e/spire_operand_test.go new file mode 100644 index 00000000..67358f68 --- /dev/null +++ b/kagenti-operator/test/e2e/spire_operand_test.go @@ -0,0 +1,336 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "os/exec" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/kagenti/operator/test/utils" +) + +// SPIRE operand CRD kinds and their plural resource names. +var spireOperandResources = []struct { + kind string + plural string + hasSpec bool + specKeys []string +}{ + {"ZeroTrustWorkloadIdentityManager", "zerotrustworkloadidentitymanagers", true, + []string{"trustDomain", "clusterName", "bundleConfigMap"}}, + {"SpiffeCSIDriver", "spiffecsidrivers", true, + []string{"agentSocketPath", "pluginName"}}, + {"SpireServer", "spireservers", true, + []string{"caSubject", "datastore", "jwtIssuer"}}, + {"SpireAgent", "spireagents", true, + []string{"nodeAttestor", "workloadAttestors"}}, + {"SpireOIDCDiscoveryProvider", "spireoidcdiscoveryproviders", true, + []string{"csiDriverName", "jwtIssuer"}}, +} + +var _ = Describe("SPIRE Operand Controller E2E", Ordered, func() { + const controllerNamespace = "kagenti-operator-system" + + BeforeAll(func() { + By("installing mock ZTWIM CRDs into Kind cluster") + _, err := utils.KubectlApplyStdin(ztwimMockCRDsYAML(), "") + Expect(err).NotTo(HaveOccurred(), "Failed to install mock ZTWIM CRDs") + + By("waiting for CRDs to be established") + for _, res := range spireOperandResources { + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "crd", + fmt.Sprintf("%s.operator.openshift.io", res.plural)) + _, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + }, 30*time.Second, 2*time.Second).Should(Succeed(), + fmt.Sprintf("CRD %s not established", res.plural)) + } + + By("deploying controller with SPIRE operand support") + Expect(utils.DeployController(controllerNamespace, projectImage)).To(Succeed(), + "Failed to deploy controller") + + By("setting SPIRE trust domain for SPIRE operand controller (no ZTWIM CR to discover from in Kind)") + setEnvCmd := exec.Command("kubectl", "set", "env", + "deployment/kagenti-operator-controller-manager", + "-n", controllerNamespace, + "KAGENTI_SPIRE_TRUST_DOMAIN=example.org") + _, err = utils.Run(setEnvCmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for controller-manager to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", + "-n", controllerNamespace, + "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .status.phase }}{{ end }}{{ end }}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Running")) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + }) + + AfterAll(func() { + By("cleaning up SPIRE operand CRs") + for _, res := range spireOperandResources { + cmd := exec.Command("kubectl", "delete", + fmt.Sprintf("%s.operator.openshift.io", res.plural), + "cluster", "--ignore-not-found") + _, _ = utils.Run(cmd) + } + + By("cleaning up mock CRDs") + for _, res := range spireOperandResources { + cmd := exec.Command("kubectl", "delete", "crd", + fmt.Sprintf("%s.operator.openshift.io", res.plural), + "--ignore-not-found") + _, _ = utils.Run(cmd) + } + + utils.UndeployController() + + By("removing manager namespace") + cmd := exec.Command("kubectl", "delete", "ns", controllerNamespace, "--ignore-not-found") + _, _ = utils.Run(cmd) + }) + + AfterEach(func() { + if CurrentSpecReport().Failed() { + cmd := exec.Command("kubectl", "logs", "-l", "control-plane=controller-manager", + "-n", controllerNamespace, "--tail=200") + logs, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n%s\n", logs) + } + } + }) + + SetDefaultEventuallyTimeout(3 * time.Minute) + SetDefaultEventuallyPollingInterval(2 * time.Second) + + It("should create all 5 SPIRE operand CRs when CRDs are present", func() { + for _, res := range spireOperandResources { + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", + fmt.Sprintf("%s.operator.openshift.io", res.plural), + "cluster", "-o", "jsonpath={.metadata.name}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(output)).To(Equal("cluster")) + }).Should(Succeed(), fmt.Sprintf("%s CR not created", res.kind)) + } + }) + + It("should label all CRs with managed-by kagenti-operator", func() { + for _, res := range spireOperandResources { + Eventually(func(g Gomega) { + output, err := utils.KubectlGetJsonpath( + fmt.Sprintf("%s.operator.openshift.io", res.plural), + "cluster", "", + "{.metadata.labels.app\\.kubernetes\\.io/managed-by}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(output)).To(Equal("kagenti-operator")) + }).Should(Succeed(), fmt.Sprintf("%s missing managed-by label", res.kind)) + } + }) + + It("should set correct ZTWIM spec", func() { + Eventually(func(g Gomega) { + clusterName, err := utils.KubectlGetJsonpath( + "zerotrustworkloadidentitymanagers.operator.openshift.io", + "cluster", "", "{.spec.clusterName}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(clusterName)).To(Equal("agent-platform")) + + bundleCM, err := utils.KubectlGetJsonpath( + "zerotrustworkloadidentitymanagers.operator.openshift.io", + "cluster", "", "{.spec.bundleConfigMap}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(bundleCM)).To(Equal("spire-bundle")) + + td, err := utils.KubectlGetJsonpath( + "zerotrustworkloadidentitymanagers.operator.openshift.io", + "cluster", "", "{.spec.trustDomain}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(td)).NotTo(BeEmpty()) + }).Should(Succeed()) + }) + + It("should set correct SpiffeCSIDriver spec", func() { + Eventually(func(g Gomega) { + socketPath, err := utils.KubectlGetJsonpath( + "spiffecsidrivers.operator.openshift.io", + "cluster", "", "{.spec.agentSocketPath}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(socketPath)).To(Equal("/run/spire/agent-sockets")) + + pluginName, err := utils.KubectlGetJsonpath( + "spiffecsidrivers.operator.openshift.io", + "cluster", "", "{.spec.pluginName}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(pluginName)).To(Equal("csi.spiffe.io")) + }).Should(Succeed()) + }) + + It("should not set trustDomain on child CRs", func() { + childResources := []string{ + "spiffecsidrivers.operator.openshift.io", + "spireservers.operator.openshift.io", + "spireagents.operator.openshift.io", + "spireoidcdiscoveryproviders.operator.openshift.io", + } + for _, res := range childResources { + Eventually(func(g Gomega) { + td, err := utils.KubectlGetJsonpath(res, "cluster", "", "{.spec.trustDomain}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(td)).To(BeEmpty(), + fmt.Sprintf("%s should not have trustDomain in spec", res)) + }).Should(Succeed()) + } + }) + + It("should reconcile drift when child CR is deleted", func() { + By("deleting SpireAgent CR") + cmd := exec.Command("kubectl", "delete", + "spireagents.operator.openshift.io", "cluster") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for SpireAgent CR to be recreated") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", + "spireagents.operator.openshift.io", "cluster", + "-o", "jsonpath={.metadata.name}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(output)).To(Equal("cluster")) + }).Should(Succeed(), "SpireAgent CR was not recreated after deletion") + + By("verifying recreated CR has managed-by label") + Eventually(func(g Gomega) { + output, err := utils.KubectlGetJsonpath( + "spireagents.operator.openshift.io", + "cluster", "", + "{.metadata.labels.app\\.kubernetes\\.io/managed-by}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(output)).To(Equal("kagenti-operator")) + }).Should(Succeed()) + }) + + It("should reconcile drift when ZTWIM spec is modified", func() { + By("patching ZTWIM bundleConfigMap to wrong value") + cmd := exec.Command("kubectl", "patch", + "zerotrustworkloadidentitymanagers.operator.openshift.io", "cluster", + "--type=merge", "-p", `{"spec":{"bundleConfigMap":"wrong-bundle"}}`) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for bundleConfigMap to be restored") + Eventually(func(g Gomega) { + output, err := utils.KubectlGetJsonpath( + "zerotrustworkloadidentitymanagers.operator.openshift.io", + "cluster", "", "{.spec.bundleConfigMap}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(output)).To(Equal("spire-bundle")) + }).Should(Succeed(), "ZTWIM bundleConfigMap was not restored after drift") + }) + + It("should reconcile drift when child spec is modified", func() { + By("patching SpiffeCSIDriver pluginName to wrong value") + cmd := exec.Command("kubectl", "patch", + "spiffecsidrivers.operator.openshift.io", "cluster", + "--type=merge", "-p", `{"spec":{"pluginName":"wrong.plugin"}}`) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("waiting for pluginName to be restored") + Eventually(func(g Gomega) { + output, err := utils.KubectlGetJsonpath( + "spiffecsidrivers.operator.openshift.io", + "cluster", "", "{.spec.pluginName}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(output)).To(Equal("csi.spiffe.io")) + }).Should(Succeed(), "SpiffeCSIDriver pluginName was not restored after drift") + }) + + It("should verify controller logs show SPIRE operand activation", func() { + cmd := exec.Command("kubectl", "logs", "-l", "control-plane=controller-manager", + "-n", controllerNamespace, "--tail=500") + logs, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + Expect(logs).To(ContainSubstring("SPIRE operand controller enabled"), + "controller should log SPIRE operand activation") + }) +}) + +// ztwimMockCRDsYAML returns minimal CRD definitions for all 5 SPIRE operand +// types. These use x-kubernetes-preserve-unknown-fields to accept any spec +// without requiring full schema validation — sufficient for Kind-based e2e +// testing where the real ZTWIM operator is absent. +func ztwimMockCRDsYAML() string { + crds := []struct { + plural string + singular string + kind string + listKind string + }{ + {"zerotrustworkloadidentitymanagers", "zerotrustworkloadidentitymanager", + "ZeroTrustWorkloadIdentityManager", "ZeroTrustWorkloadIdentityManagerList"}, + {"spiffecsidrivers", "spiffecsidriver", + "SpiffeCSIDriver", "SpiffeCSIDriverList"}, + {"spireservers", "spireserver", + "SpireServer", "SpireServerList"}, + {"spireagents", "spireagent", + "SpireAgent", "SpireAgentList"}, + {"spireoidcdiscoveryproviders", "spireoidcdiscoveryprovider", + "SpireOIDCDiscoveryProvider", "SpireOIDCDiscoveryProviderList"}, + } + + var sb strings.Builder + for i, crd := range crds { + if i > 0 { + sb.WriteString("---\n") + } + fmt.Fprintf(&sb, `apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: %s.operator.openshift.io +spec: + group: operator.openshift.io + names: + kind: %s + listKind: %s + plural: %s + singular: %s + scope: Cluster + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true +`, crd.plural, crd.kind, crd.listKind, crd.plural, crd.singular) + } + return sb.String() +}