From ac91e7014c03ad29f6e06521eb00ef07bd8ba1de Mon Sep 17 00:00:00 2001 From: Alan Cha Date: Fri, 10 Jul 2026 09:21:52 -0400 Subject: [PATCH 01/12] docs(values): clarify spiffe.enabled vs spiffe.operatorAuth.enabled and jwtAudience Add comments to values.yaml explaining the semantic distinction between the two flags and what jwtAudience actually represents in the SPIFFE/Keycloak flow. Signed-off-by: Alan Cha Assisted-By: Claude (Anthropic AI) Signed-off-by: Alan Cha --- charts/kagenti-operator/values.yaml | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/charts/kagenti-operator/values.yaml b/charts/kagenti-operator/values.yaml index acdde623..682d8feb 100644 --- a/charts/kagenti-operator/values.yaml +++ b/charts/kagenti-operator/values.yaml @@ -190,15 +190,26 @@ sigstore: configMapKey: "trusted-root.json" # [SPIFFE OPERATOR AUTH]: Operator authentication to Keycloak using JWT-SVID -# When enabled, the operator uses its SPIFFE identity (JWT-SVID) to authenticate -# to Keycloak instead of admin credentials. Requires Keycloak SPIFFE IdP configured. +# spiffe.enabled — master switch indicating that SPIRE is present in the cluster +# and SPIFFE workload identities are available. Set to true whenever --with-spire +# is used. Features that consume SPIFFE identities (e.g. verified-fetch, operatorAuth) +# each have their own sub-flag; this top-level switch guards them all. +# +# spiffe.operatorAuth — controls how the operator authenticates to the Keycloak +# Admin API when registering agent OAuth clients. When disabled (default), the +# operator uses admin credentials from keycloak-admin-secret. When enabled, the +# operator uses its own SPIFFE JWT-SVID, eliminating the need for that secret. +# This is independent of how agent/tool workloads authenticate (see authBridge +# clientAuthType in the kagenti chart). spiffe: enabled: false operatorAuth: enabled: false - # JWT audience must match Keycloak's realm issuer URL exactly. - # If not set, defaults to: {{ .Values.keycloak.publicUrl }}/realms/{{ .Values.keycloak.realm }} - # Check Keycloak's .well-known/openid-configuration for the correct issuer value. + # jwtAudience — the audience value embedded in the JWT-SVID that SPIRE mints for + # the operator. Must exactly match the issuer URL that Keycloak's SPIFFE Identity + # Provider is configured to accept (typically the Keycloak realm's public URL: + # http:///realms/). Check Keycloak's + # .well-known/openid-configuration "issuer" field for the correct value. jwtAudience: "" # Path to JWT-SVID file written by spiffe-helper sidecar jwtSVIDPath: "/opt/jwt_svid.token" From 3d59b722ae75575210635988c8a01d8afa796127 Mon Sep 17 00:00:00 2001 From: Alan Cha Date: Fri, 10 Jul 2026 10:19:22 -0400 Subject: [PATCH 02/12] docs(values): document jwtAudience audience mismatch pitfall Keycloak's FederatedJWTClientValidator checks the JWT-SVID aud claim against the realm's OIDC issuer URL (always the external/public URL). Using the internal k8s service address causes a silent audience mismatch and invalid_client_credentials. Document the correct value and how to find it. Signed-off-by: Alan Cha Assisted-By: Claude (Anthropic AI) Signed-off-by: Alan Cha --- charts/kagenti-operator/values.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/charts/kagenti-operator/values.yaml b/charts/kagenti-operator/values.yaml index 682d8feb..8321adbf 100644 --- a/charts/kagenti-operator/values.yaml +++ b/charts/kagenti-operator/values.yaml @@ -205,11 +205,19 @@ spiffe: enabled: false operatorAuth: enabled: false - # jwtAudience — the audience value embedded in the JWT-SVID that SPIRE mints for - # the operator. Must exactly match the issuer URL that Keycloak's SPIFFE Identity - # Provider is configured to accept (typically the Keycloak realm's public URL: - # http:///realms/). Check Keycloak's - # .well-known/openid-configuration "issuer" field for the correct value. + # jwtAudience — the audience claim SPIRE will embed in the JWT-SVID it mints for + # the operator. Keycloak's FederatedJWTClientValidator checks that this value + # exactly matches the realm's issuer URL from its OIDC discovery document. + # + # IMPORTANT: this must be the PUBLIC/EXTERNAL Keycloak URL, not the in-cluster + # service address. Using the internal address (e.g. http://keycloak-service.keycloak.svc) + # causes an audience mismatch because Keycloak's issuer is always the external URL + # it was configured with (keycloak.publicUrl). + # + # Correct: http://keycloak.example.com/realms/kagenti (matches .well-known issuer) + # Wrong: http://keycloak-service.keycloak.svc:8080/realms/kagenti (internal, rejected) + # + # To find the correct value: curl http:///realms//.well-known/openid-configuration | jq .issuer jwtAudience: "" # Path to JWT-SVID file written by spiffe-helper sidecar jwtSVIDPath: "/opt/jwt_svid.token" From 62f8a1b6bbe22562d02bd9b1da7397021d1fdf5c Mon Sep 17 00:00:00 2001 From: Alan Cha Date: Fri, 10 Jul 2026 10:37:14 -0400 Subject: [PATCH 03/12] fix(charts): remove jwtAudience value, always derive from keycloak.publicUrl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit jwtAudience was always going to be keycloak.publicUrl/realms/ — the issuer URL of the Keycloak realm the operator authenticates to. There is no meaningful reason to configure it independently. Remove the user-facing field and derive the audience directly in the template. keycloak.publicUrl is already required for other reasons (UI, authbridge) so this does not add a new dependency. Fail at helm install time with a clear message if keycloak.publicUrl is not set. Signed-off-by: Alan Cha Assisted-By: Claude (Anthropic AI) Signed-off-by: Alan Cha --- .../templates/manager/configmap-spiffe-helper.yaml | 14 +++++++++++++- charts/kagenti-operator/values.yaml | 14 -------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/charts/kagenti-operator/templates/manager/configmap-spiffe-helper.yaml b/charts/kagenti-operator/templates/manager/configmap-spiffe-helper.yaml index 75cb7079..075b3b16 100644 --- a/charts/kagenti-operator/templates/manager/configmap-spiffe-helper.yaml +++ b/charts/kagenti-operator/templates/manager/configmap-spiffe-helper.yaml @@ -17,8 +17,20 @@ data: cert_dir = "" svid_file_name = "" svid_bundle_file_name = "" + {{- /* + jwt_audience must equal the Keycloak realm's issuer URL — the string Keycloak + advertises in its /.well-known/openid-configuration. It is always derived from + keycloak.publicUrl (the external URL Keycloak was configured with). The internal + k8s service address must NOT be used because Keycloak's FederatedJWTClientValidator + checks the aud claim via string equality against its own issuer, which is always + the external URL. + */ -}} + {{- $audience := printf "%s/realms/%s" .Values.keycloak.publicUrl .Values.keycloak.realm -}} + {{- if not (hasPrefix "http" $audience) -}} + {{- fail "keycloak.publicUrl must be set to the external Keycloak URL (e.g. http://keycloak.example.com) when spiffe.operatorAuth.enabled=true" -}} + {{- end -}} jwt_svids = [{ - jwt_audience = "{{ required "spiffe.operatorAuth.jwtAudience must be set when spiffe.operatorAuth.enabled=true (e.g. http://keycloak.example.com/realms/kagenti)" .Values.spiffe.operatorAuth.jwtAudience }}" + jwt_audience = "{{ $audience }}" jwt_svid_file_name = "/opt/jwt_svid.token" }] {{- end }} diff --git a/charts/kagenti-operator/values.yaml b/charts/kagenti-operator/values.yaml index 8321adbf..5a5b08b7 100644 --- a/charts/kagenti-operator/values.yaml +++ b/charts/kagenti-operator/values.yaml @@ -205,20 +205,6 @@ spiffe: enabled: false operatorAuth: enabled: false - # jwtAudience — the audience claim SPIRE will embed in the JWT-SVID it mints for - # the operator. Keycloak's FederatedJWTClientValidator checks that this value - # exactly matches the realm's issuer URL from its OIDC discovery document. - # - # IMPORTANT: this must be the PUBLIC/EXTERNAL Keycloak URL, not the in-cluster - # service address. Using the internal address (e.g. http://keycloak-service.keycloak.svc) - # causes an audience mismatch because Keycloak's issuer is always the external URL - # it was configured with (keycloak.publicUrl). - # - # Correct: http://keycloak.example.com/realms/kagenti (matches .well-known issuer) - # Wrong: http://keycloak-service.keycloak.svc:8080/realms/kagenti (internal, rejected) - # - # To find the correct value: curl http:///realms//.well-known/openid-configuration | jq .issuer - jwtAudience: "" # Path to JWT-SVID file written by spiffe-helper sidecar jwtSVIDPath: "/opt/jwt_svid.token" spiffeHelper: From d02fe8020f1b19e9f62af270b1f8dfa813765b70 Mon Sep 17 00:00:00 2001 From: Alan Cha Date: Sun, 12 Jul 2026 09:21:18 -0400 Subject: [PATCH 04/12] fix(charts): use v0.5.0-rc.3 as spiffe-helper default tag v0.5.0-rc.3 is the latest versioned spiffe-helper image tag confirmed to exist in ghcr.io. Previous tag (v0.6.0-alpha.4) was a GitHub release tag but the corresponding container image was never published. Update comment to warn that not all release tags produce container images. Signed-off-by: Alan Cha Assisted-By: Claude (Anthropic AI) Signed-off-by: Alan Cha --- charts/kagenti-operator/values.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/kagenti-operator/values.yaml b/charts/kagenti-operator/values.yaml index 5a5b08b7..23d77c52 100644 --- a/charts/kagenti-operator/values.yaml +++ b/charts/kagenti-operator/values.yaml @@ -210,7 +210,9 @@ spiffe: spiffeHelper: image: repository: ghcr.io/kagenti/kagenti-extensions/spiffe-helper - tag: v0.6.0-alpha.4 + # v0.5.0-rc.3 is the latest published versioned tag in the kagenti-extensions registry. + # Note: not all GitHub release tags produce container images — check ghcr.io before bumping. + tag: v0.5.0-rc.3 pullPolicy: IfNotPresent # Feature gates — highest-priority layer in the injection precedence chain. From aae42d322b5dd4878406ee623e5f68afc12f6335 Mon Sep 17 00:00:00 2001 From: Alan Cha Date: Sun, 12 Jul 2026 09:19:56 -0400 Subject: [PATCH 05/12] fix(controller): skip credential secret creation in federated-jwt mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In federated-jwt mode, agents authenticate to Keycloak using their SPIFFE identity (JWT-SVID) directly via the workload API socket. The operator still registers the Keycloak client with federated-jwt authenticator type, but no credential Secret is needed — AuthBridge reads JWT-SVIDs directly, not from a mounted client-secret file. Previously the operator wrote a Kubernetes Secret containing the Keycloak- generated client secret even in federated-jwt mode, even though nothing ever reads it. This is a hygiene issue (unnecessary secrets in agent namespaces) and a defense-in-depth gap (secret exists even if Keycloak correctly rejects it). Skipping creation removes the noise. Signed-off-by: Alan Cha Assisted-By: Claude (Anthropic AI) Signed-off-by: Alan Cha --- .../controller/clientregistration_controller.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/kagenti-operator/internal/controller/clientregistration_controller.go b/kagenti-operator/internal/controller/clientregistration_controller.go index 9efcb04a..c62df3cc 100644 --- a/kagenti-operator/internal/controller/clientregistration_controller.go +++ b/kagenti-operator/internal/controller/clientregistration_controller.go @@ -370,10 +370,19 @@ func (r *ClientRegistrationReconciler) reconcileOne( "clientId", clientID) } + // In federated-jwt mode, agents authenticate using their SPIFFE identity directly. + // No credential secret is needed — AuthBridge reads JWT-SVIDs from the workload + // API socket, not from a mounted secret. Skipping secret creation avoids leaving + // unused Keycloak client secrets in agent namespaces. secretName := keycloakClientCredentialsSecretName(ns, workloadName) - if err := r.ensureClientCredentialsSecret(ctx, owner, secretName, clientID, clientSecret); err != nil { - logger.Error(err, "ensure client credentials secret") - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + if authType != "federated-jwt" { + if err := r.ensureClientCredentialsSecret(ctx, owner, secretName, clientID, clientSecret); err != nil { + logger.Error(err, "ensure client credentials secret") + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + } else { + logger.V(1).Info("skipping credential secret (federated-jwt mode — AuthBridge uses SPIFFE identity directly)", + "workload", workloadName, "namespace", ns) } if err := patchTemplate(ctx); err != nil { From 22aaa58fb6b9bb664beb47f96a7a4830ec53a8f2 Mon Sep 17 00:00:00 2001 From: Alan Cha Date: Sun, 12 Jul 2026 09:54:53 -0400 Subject: [PATCH 06/12] fix(charts): remove noisy comment from spiffe-helper tag Signed-off-by: Alan Cha Assisted-By: Claude (Anthropic AI) Signed-off-by: Alan Cha --- charts/kagenti-operator/values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/kagenti-operator/values.yaml b/charts/kagenti-operator/values.yaml index 23d77c52..152309da 100644 --- a/charts/kagenti-operator/values.yaml +++ b/charts/kagenti-operator/values.yaml @@ -210,8 +210,6 @@ spiffe: spiffeHelper: image: repository: ghcr.io/kagenti/kagenti-extensions/spiffe-helper - # v0.5.0-rc.3 is the latest published versioned tag in the kagenti-extensions registry. - # Note: not all GitHub release tags produce container images — check ghcr.io before bumping. tag: v0.5.0-rc.3 pullPolicy: IfNotPresent From 573b1f766f4e8cde19ed222a5b99c92d8794040b Mon Sep 17 00:00:00 2001 From: Alan Cha Date: Sun, 12 Jul 2026 10:09:40 -0400 Subject: [PATCH 07/12] revert(charts): remove spiffeHelper.image values, hardcode :latest in template The spiffeHelper.image.* values block was introduced to pin the sidecar image, but ghcr.io/kagenti/kagenti-extensions/spiffe-helper is not published by the kagenti-extensions automated pipeline and the pinned tag (v0.6.0-alpha.4) does not exist in the registry. Remove the values block entirely and hardcode :latest in the template, which is the only published image. A separate issue will track the correct long-term approach. Signed-off-by: Alan Cha Assisted-By: Claude (Anthropic AI) Signed-off-by: Alan Cha --- charts/kagenti-operator/templates/manager/manager.yaml | 4 ++-- charts/kagenti-operator/values.yaml | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/charts/kagenti-operator/templates/manager/manager.yaml b/charts/kagenti-operator/templates/manager/manager.yaml index 5a9fc57b..d7fa042f 100644 --- a/charts/kagenti-operator/templates/manager/manager.yaml +++ b/charts/kagenti-operator/templates/manager/manager.yaml @@ -193,8 +193,8 @@ spec: {{- end }} {{- if and .Values.spiffe .Values.spiffe.enabled .Values.spiffe.operatorAuth .Values.spiffe.operatorAuth.enabled }} - name: spiffe-helper - image: {{ .Values.spiffe.operatorAuth.spiffeHelper.image.repository }}:{{ .Values.spiffe.operatorAuth.spiffeHelper.image.tag }} - imagePullPolicy: {{ .Values.spiffe.operatorAuth.spiffeHelper.image.pullPolicy | default "IfNotPresent" }} + image: ghcr.io/kagenti/kagenti-extensions/spiffe-helper:latest + imagePullPolicy: IfNotPresent args: - "-config" - "/etc/spiffe-helper/config.hcl" diff --git a/charts/kagenti-operator/values.yaml b/charts/kagenti-operator/values.yaml index 152309da..e941c3c8 100644 --- a/charts/kagenti-operator/values.yaml +++ b/charts/kagenti-operator/values.yaml @@ -207,11 +207,6 @@ spiffe: enabled: false # Path to JWT-SVID file written by spiffe-helper sidecar jwtSVIDPath: "/opt/jwt_svid.token" - spiffeHelper: - image: - repository: ghcr.io/kagenti/kagenti-extensions/spiffe-helper - tag: v0.5.0-rc.3 - pullPolicy: IfNotPresent # Feature gates — highest-priority layer in the injection precedence chain. # Set globalEnabled to false to disable ALL sidecar injection (kill switch). From f07ac4350f61c4a711daa9e9d31f8169e43bcfe8 Mon Sep 17 00:00:00 2001 From: Alan Cha Date: Sun, 12 Jul 2026 11:01:39 -0400 Subject: [PATCH 08/12] fix(controller): also skip patchTemplate annotation in federated-jwt mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When skipping credential secret creation in federated-jwt mode, also skip annotating the pod template with the secret name. The webhook mounts the credential secret based on that annotation — if we skip secret creation but still set the annotation, pods get stuck in Init because the secret does not exist. In federated-jwt mode, AuthBridge reads JWT-SVIDs from the SPIFFE workload API socket directly and does not need the credential secret at all. Signed-off-by: Alan Cha Assisted-By: Claude (Anthropic AI) Signed-off-by: Alan Cha --- .../controller/clientregistration_controller.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/kagenti-operator/internal/controller/clientregistration_controller.go b/kagenti-operator/internal/controller/clientregistration_controller.go index c62df3cc..15ad4e44 100644 --- a/kagenti-operator/internal/controller/clientregistration_controller.go +++ b/kagenti-operator/internal/controller/clientregistration_controller.go @@ -380,16 +380,20 @@ func (r *ClientRegistrationReconciler) reconcileOne( logger.Error(err, "ensure client credentials secret") return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } + // Annotate the pod template with the secret name so the webhook mounts it. + if err := patchTemplate(ctx); err != nil { + logger.Error(err, "patch workload pod template") + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } } else { - logger.V(1).Info("skipping credential secret (federated-jwt mode — AuthBridge uses SPIFFE identity directly)", + // In federated-jwt mode, AuthBridge reads JWT-SVIDs directly from the SPIFFE + // workload API socket — no credential secret is needed or created. Skip both the + // secret and the pod template annotation so the webhook does not try to mount a + // non-existent secret. + logger.V(1).Info("skipping credential secret and template patch (federated-jwt — AuthBridge uses SPIFFE identity directly)", "workload", workloadName, "namespace", ns) } - if err := patchTemplate(ctx); err != nil { - logger.Error(err, "patch workload pod template") - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil - } - logger.Info("operator client registration applied", "workload", workloadName, "namespace", ns, "secret", secretName) return ctrl.Result{}, nil From 8cac96cab8db4a717f0c6884cbe7ddf94bb6d93d Mon Sep 17 00:00:00 2001 From: Alan Cha Date: Mon, 13 Jul 2026 06:40:55 -0400 Subject: [PATCH 09/12] fix(webhook): skip credential annotation in federated-jwt mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The authbridge webhook pre-populates the Keycloak client-credentials annotation so the credential Secret volume is ready before the controller reconciles. In federated-jwt mode, no credential Secret is created — AuthBridge reads JWT-SVIDs from the SPIFFE workload API socket directly. Injecting the volume mount for a non-existent Secret left pods stuck in Init:0/1. Fix: read CLIENT_AUTH_TYPE from authbridge-config before pre-populating the annotation, and skip it when federated-jwt is configured. Signed-off-by: Alan Cha Assisted-By: Claude (Anthropic AI) Signed-off-by: Alan Cha --- .../webhook/v1alpha1/authbridge_webhook.go | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/kagenti-operator/internal/webhook/v1alpha1/authbridge_webhook.go b/kagenti-operator/internal/webhook/v1alpha1/authbridge_webhook.go index c6cd01fd..9f186f64 100644 --- a/kagenti-operator/internal/webhook/v1alpha1/authbridge_webhook.go +++ b/kagenti-operator/internal/webhook/v1alpha1/authbridge_webhook.go @@ -96,16 +96,23 @@ func (w *AuthBridgeWebhook) Handle(ctx context.Context, req admission.Request) a // kubelet will wait for the Secret to appear and mount it — no pod restart required. // Without this, the first pod comes up with an empty /shared/ and envoy returns 503 // "identity not yet configured (credentials pending)" until the user deletes the pod. + // + // Skip in federated-jwt mode: AuthBridge reads JWT-SVIDs from the SPIFFE workload API + // socket directly and no credential Secret is created or needed. Injecting the volume + // mount for a non-existent Secret would leave pods stuck in Init. if pod.Annotations[injector.AnnotationKeycloakClientSecretName] == "" && clientreg.WorkloadWantsOperatorClientReg(pod.Labels, w.Mutator.GetFeatureGates().InjectTools) { - if pod.Annotations == nil { - pod.Annotations = map[string]string{} + nsConfig, _ := injector.ReadNamespaceConfig(ctx, w.Mutator.APIReader, req.Namespace) + if nsConfig == nil || nsConfig.ClientAuthType != injector.ClientAuthTypeFederatedJWT { + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations[injector.AnnotationKeycloakClientSecretName] = + clientreg.KeycloakClientCredentialsSecretName(req.Namespace, resourceName) + authbridgelog.Info("pre-populated Keycloak client credentials annotation", + "namespace", req.Namespace, "name", resourceName, + "secret", pod.Annotations[injector.AnnotationKeycloakClientSecretName]) } - pod.Annotations[injector.AnnotationKeycloakClientSecretName] = - clientreg.KeycloakClientCredentialsSecretName(req.Namespace, resourceName) - authbridgelog.Info("pre-populated Keycloak client credentials annotation", - "namespace", req.Namespace, "name", resourceName, - "secret", pod.Annotations[injector.AnnotationKeycloakClientSecretName]) } // Check if already injected (idempotency / reinvocation) From 77598e9078e40ba8569b9d63bda1d64027066b2f Mon Sep 17 00:00:00 2001 From: Alan Cha Date: Mon, 13 Jul 2026 07:06:55 -0400 Subject: [PATCH 10/12] fix(charts): restore spiffeHelper.image values block with latest as default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hardcoding :latest in the template was worse — it removed the ability to override the image via Helm values without forking the chart. Restore the templated approach with tag: latest as the default, which is the only reliably-published tag. Signed-off-by: Alan Cha Assisted-By: Claude (Anthropic AI) Signed-off-by: Alan Cha --- charts/kagenti-operator/templates/manager/manager.yaml | 4 ++-- charts/kagenti-operator/values.yaml | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/charts/kagenti-operator/templates/manager/manager.yaml b/charts/kagenti-operator/templates/manager/manager.yaml index d7fa042f..e97f31f6 100644 --- a/charts/kagenti-operator/templates/manager/manager.yaml +++ b/charts/kagenti-operator/templates/manager/manager.yaml @@ -193,8 +193,8 @@ spec: {{- end }} {{- if and .Values.spiffe .Values.spiffe.enabled .Values.spiffe.operatorAuth .Values.spiffe.operatorAuth.enabled }} - name: spiffe-helper - image: ghcr.io/kagenti/kagenti-extensions/spiffe-helper:latest - imagePullPolicy: IfNotPresent + image: {{ .Values.spiffe.operatorAuth.spiffeHelper.image.repository | default "ghcr.io/kagenti/kagenti-extensions/spiffe-helper" }}:{{ .Values.spiffe.operatorAuth.spiffeHelper.image.tag | default "latest" }} + imagePullPolicy: {{ .Values.spiffe.operatorAuth.spiffeHelper.image.pullPolicy | default "IfNotPresent" }} args: - "-config" - "/etc/spiffe-helper/config.hcl" diff --git a/charts/kagenti-operator/values.yaml b/charts/kagenti-operator/values.yaml index e941c3c8..a9b130ff 100644 --- a/charts/kagenti-operator/values.yaml +++ b/charts/kagenti-operator/values.yaml @@ -207,6 +207,11 @@ spiffe: enabled: false # Path to JWT-SVID file written by spiffe-helper sidecar jwtSVIDPath: "/opt/jwt_svid.token" + spiffeHelper: + image: + repository: ghcr.io/kagenti/kagenti-extensions/spiffe-helper + tag: latest + pullPolicy: IfNotPresent # Feature gates — highest-priority layer in the injection precedence chain. # Set globalEnabled to false to disable ALL sidecar injection (kill switch). From 2e6ad42b3dabcab9ca888edfb759ed00be2c5165 Mon Sep 17 00:00:00 2001 From: Alan Cha Date: Mon, 13 Jul 2026 07:14:06 -0400 Subject: [PATCH 11/12] revert: back to hardcoded spiffe-helper:latest, remove values block Signed-off-by: Alan Cha Assisted-By: Claude (Anthropic AI) Signed-off-by: Alan Cha --- charts/kagenti-operator/templates/manager/manager.yaml | 4 ++-- charts/kagenti-operator/values.yaml | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/charts/kagenti-operator/templates/manager/manager.yaml b/charts/kagenti-operator/templates/manager/manager.yaml index e97f31f6..d7fa042f 100644 --- a/charts/kagenti-operator/templates/manager/manager.yaml +++ b/charts/kagenti-operator/templates/manager/manager.yaml @@ -193,8 +193,8 @@ spec: {{- end }} {{- if and .Values.spiffe .Values.spiffe.enabled .Values.spiffe.operatorAuth .Values.spiffe.operatorAuth.enabled }} - name: spiffe-helper - image: {{ .Values.spiffe.operatorAuth.spiffeHelper.image.repository | default "ghcr.io/kagenti/kagenti-extensions/spiffe-helper" }}:{{ .Values.spiffe.operatorAuth.spiffeHelper.image.tag | default "latest" }} - imagePullPolicy: {{ .Values.spiffe.operatorAuth.spiffeHelper.image.pullPolicy | default "IfNotPresent" }} + image: ghcr.io/kagenti/kagenti-extensions/spiffe-helper:latest + imagePullPolicy: IfNotPresent args: - "-config" - "/etc/spiffe-helper/config.hcl" diff --git a/charts/kagenti-operator/values.yaml b/charts/kagenti-operator/values.yaml index a9b130ff..e941c3c8 100644 --- a/charts/kagenti-operator/values.yaml +++ b/charts/kagenti-operator/values.yaml @@ -207,11 +207,6 @@ spiffe: enabled: false # Path to JWT-SVID file written by spiffe-helper sidecar jwtSVIDPath: "/opt/jwt_svid.token" - spiffeHelper: - image: - repository: ghcr.io/kagenti/kagenti-extensions/spiffe-helper - tag: latest - pullPolicy: IfNotPresent # Feature gates — highest-priority layer in the injection precedence chain. # Set globalEnabled to false to disable ALL sidecar injection (kill switch). From 0532b068aa6f48b3e363c08bed25f396ee617a0b Mon Sep 17 00:00:00 2001 From: Alan Cha Date: Mon, 13 Jul 2026 08:42:54 -0400 Subject: [PATCH 12/12] feat(operator): fetch JWT-SVID directly via go-spiffe SDK, remove spiffe-helper sidecar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spiffe-helper sidecar approach used an abandoned standalone image from the kagenti-extensions pipeline (removed April 2026 when spiffe-helper was bundled into authbridge). The authbridge-proxy itself already uses the go-spiffe WorkloadAPI client in-process — this change makes the operator consistent with that architecture. Changes: - clientregistration_controller.go: replace JWTSVIDPath + os.ReadFile with workloadapi.New() + FetchJWTSVID(). Uses the same socket as verifiedFetch. Removes path traversal validation (no longer needed without file I/O). - cmd/main.go: remove --jwt-svid-path flag; reuse --verified-fetch-spiffe-socket for JWT-SVID fetching via SpiffeSocket field on the reconciler. - manager.yaml: remove spiffe-helper container, jwt-svid emptyDir volume, spiffe-helper-config volume mount. spiffe-workload-api CSI volume remains (shared with verifiedFetch). - configmap-spiffe-helper.yaml: deleted (no longer needed). - values.yaml: remove jwtSVIDPath value. Signed-off-by: Alan Cha Assisted-By: Claude (Anthropic AI) Signed-off-by: Alan Cha --- .../manager/configmap-spiffe-helper.yaml | 36 ----------- .../templates/manager/manager.yaml | 45 -------------- charts/kagenti-operator/values.yaml | 2 - kagenti-operator/cmd/main.go | 8 +-- .../clientregistration_controller.go | 62 ++++++++++--------- 5 files changed, 36 insertions(+), 117 deletions(-) delete mode 100644 charts/kagenti-operator/templates/manager/configmap-spiffe-helper.yaml diff --git a/charts/kagenti-operator/templates/manager/configmap-spiffe-helper.yaml b/charts/kagenti-operator/templates/manager/configmap-spiffe-helper.yaml deleted file mode 100644 index 075b3b16..00000000 --- a/charts/kagenti-operator/templates/manager/configmap-spiffe-helper.yaml +++ /dev/null @@ -1,36 +0,0 @@ -{{- if and .Values.spiffe .Values.spiffe.enabled .Values.spiffe.operatorAuth .Values.spiffe.operatorAuth.enabled }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: operator-spiffe-helper-config - namespace: {{ .Release.Namespace }} - labels: - {{- include "chart.labels" . | nindent 4 }} -data: - config.hcl: | - agent_address = "/spiffe-workload-api/spire-agent.sock" - # cmd/cmd_args fire on X.509 SVID renewals only, not JWT SVIDs. - # File permissions are handled by running spiffe-helper as the same UID - # as the manager container (65532) so it can read the 600-mode file. - renew_signal = "" - cert_dir = "" - svid_file_name = "" - svid_bundle_file_name = "" - {{- /* - jwt_audience must equal the Keycloak realm's issuer URL — the string Keycloak - advertises in its /.well-known/openid-configuration. It is always derived from - keycloak.publicUrl (the external URL Keycloak was configured with). The internal - k8s service address must NOT be used because Keycloak's FederatedJWTClientValidator - checks the aud claim via string equality against its own issuer, which is always - the external URL. - */ -}} - {{- $audience := printf "%s/realms/%s" .Values.keycloak.publicUrl .Values.keycloak.realm -}} - {{- if not (hasPrefix "http" $audience) -}} - {{- fail "keycloak.publicUrl must be set to the external Keycloak URL (e.g. http://keycloak.example.com) when spiffe.operatorAuth.enabled=true" -}} - {{- end -}} - jwt_svids = [{ - jwt_audience = "{{ $audience }}" - jwt_svid_file_name = "/opt/jwt_svid.token" - }] -{{- end }} diff --git a/charts/kagenti-operator/templates/manager/manager.yaml b/charts/kagenti-operator/templates/manager/manager.yaml index d7fa042f..92f38458 100644 --- a/charts/kagenti-operator/templates/manager/manager.yaml +++ b/charts/kagenti-operator/templates/manager/manager.yaml @@ -118,7 +118,6 @@ spec: {{- end }} {{- if and .Values.spiffe .Values.spiffe.enabled .Values.spiffe.operatorAuth .Values.spiffe.operatorAuth.enabled }} - "--use-spiffe-auth=true" - - "--jwt-svid-path={{ .Values.spiffe.operatorAuth.jwtSVIDPath | default "/opt/jwt_svid.token" }}" - "--operator-client-id=spiffe://{{ .Values.signatureVerification.spireTrustDomain | default "localtest.me" }}/ns/{{ .Release.Namespace }}/sa/{{ .Values.controllerManager.serviceAccountName }}" {{- end }} command: @@ -186,42 +185,6 @@ spec: mountPath: /spiffe-workload-api readOnly: true {{- end }} - {{- if and .Values.spiffe .Values.spiffe.enabled .Values.spiffe.operatorAuth .Values.spiffe.operatorAuth.enabled }} - - name: jwt-svid - mountPath: /opt - readOnly: true - {{- end }} - {{- if and .Values.spiffe .Values.spiffe.enabled .Values.spiffe.operatorAuth .Values.spiffe.operatorAuth.enabled }} - - name: spiffe-helper - image: ghcr.io/kagenti/kagenti-extensions/spiffe-helper:latest - imagePullPolicy: IfNotPresent - args: - - "-config" - - "/etc/spiffe-helper/config.hcl" - volumeMounts: - - name: spiffe-workload-api - mountPath: /spiffe-workload-api - readOnly: true - - name: spiffe-helper-config - mountPath: /etc/spiffe-helper - readOnly: true - - name: jwt-svid - mountPath: /opt - securityContext: - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 65532 - capabilities: - drop: - - ALL - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - {{- end }} securityContext: {{- toYaml .Values.controllerManager.securityContext | nindent 8 }} serviceAccountName: {{ .Values.controllerManager.serviceAccountName }} @@ -251,11 +214,3 @@ spec: driver: "csi.spiffe.io" readOnly: true {{- end }} - {{- if and .Values.spiffe .Values.spiffe.enabled .Values.spiffe.operatorAuth .Values.spiffe.operatorAuth.enabled }} - - name: spiffe-helper-config - configMap: - name: operator-spiffe-helper-config - - name: jwt-svid - emptyDir: - medium: Memory - {{- end }} diff --git a/charts/kagenti-operator/values.yaml b/charts/kagenti-operator/values.yaml index e941c3c8..c612df12 100644 --- a/charts/kagenti-operator/values.yaml +++ b/charts/kagenti-operator/values.yaml @@ -205,8 +205,6 @@ spiffe: enabled: false operatorAuth: enabled: false - # Path to JWT-SVID file written by spiffe-helper sidecar - jwtSVIDPath: "/opt/jwt_svid.token" # Feature gates — highest-priority layer in the injection precedence chain. # Set globalEnabled to false to disable ALL sidecar injection (kill switch). diff --git a/kagenti-operator/cmd/main.go b/kagenti-operator/cmd/main.go index 1be8a4eb..e9c42c20 100644 --- a/kagenti-operator/cmd/main.go +++ b/kagenti-operator/cmd/main.go @@ -135,7 +135,6 @@ func main() { var credentialWaitTimeout string var enableAuthbridgeConfig bool var useSpiffeAuth bool - var jwtSVIDPath string var operatorClientID string flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ @@ -211,9 +210,8 @@ func main() { flag.BoolVar(&enableAuthbridgeConfig, "enable-authbridge-config", true, "Reconcile authbridge-config ConfigMap in namespaces labeled kagenti-enabled=true") flag.BoolVar(&useSpiffeAuth, "use-spiffe-auth", false, - "Use JWT-SVID authentication for Keycloak client registration instead of admin credentials") - flag.StringVar(&jwtSVIDPath, "jwt-svid-path", "/opt/jwt_svid.token", - "Path to JWT-SVID file written by spiffe-helper sidecar (used when --use-spiffe-auth=true)") + "Use JWT-SVID authentication for Keycloak client registration instead of admin credentials. "+ + "JWT-SVIDs are fetched directly from the SPIRE workload API (--verified-fetch-spiffe-socket).") flag.StringVar(&operatorClientID, "operator-client-id", "", "Operator SPIFFE ID (e.g. spiffe:///ns//sa/), used when --use-spiffe-auth=true") @@ -711,7 +709,7 @@ func main() { SpireTrustDomain: spireTrustDomain, KeycloakAdminTokenCache: &keycloak.CachedAdminTokenProvider{}, UseSpiffeAuth: useSpiffeAuth, - JWTSVIDPath: jwtSVIDPath, + SpiffeSocket: verifiedFetchSpiffeSocket, OperatorClientID: operatorClientID, Recorder: mgr.GetEventRecorderFor("clientregistration"), //nolint:staticcheck }).SetupWithManager(mgr); err != nil { diff --git a/kagenti-operator/internal/controller/clientregistration_controller.go b/kagenti-operator/internal/controller/clientregistration_controller.go index 15ad4e44..086b2083 100644 --- a/kagenti-operator/internal/controller/clientregistration_controller.go +++ b/kagenti-operator/internal/controller/clientregistration_controller.go @@ -10,12 +10,13 @@ package controller import ( "context" "fmt" - "os" - "path/filepath" "sort" "strings" "time" + "github.com/spiffe/go-spiffe/v2/svid/jwtsvid" + "github.com/spiffe/go-spiffe/v2/workloadapi" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -80,15 +81,16 @@ type ClientRegistrationReconciler struct { KeycloakAdminTokenCache *keycloak.CachedAdminTokenProvider // UseSpiffeAuth enables JWT-SVID authentication instead of admin credentials. - // When true, the operator authenticates to Keycloak with its JWT-SVID and uses - // the Admin API with manage-clients role. When false, uses admin credentials. + // When true, the operator fetches a JWT-SVID from SPIRE via the go-spiffe SDK + // and uses it with the Keycloak Admin API (manage-clients role only). + // When false, uses admin credentials. UseSpiffeAuth bool - // JWTSVIDPath is the file path to read the operator's JWT-SVID from. - // Only used when UseSpiffeAuth is true. Default: /opt/jwt_svid.token - JWTSVIDPath string + // SpiffeSocket is the SPIRE workload API socket address used to fetch JWT-SVIDs. + // Only used when UseSpiffeAuth is true. Defaults to the verified-fetch socket. + SpiffeSocket string - // OperatorClientID is the operator's SPIFFE ID (e.g., spiffe://localtest.me/ns/kagenti-operator-system/sa/...). + // OperatorClientID is the operator's SPIFFE ID (e.g., spiffe://localtest.me/ns/kagenti-system/sa/controller-manager). // Only used when UseSpiffeAuth is true. OperatorClientID string @@ -269,8 +271,6 @@ func (r *ClientRegistrationReconciler) reconcileOne( // Authenticate to Keycloak: use JWT-SVID if enabled, otherwise admin credentials if r.UseSpiffeAuth { - // SPIFFE authentication path - // Validate OperatorClientID before file I/O to fail fast on misconfiguration if r.OperatorClientID == "" { err := fmt.Errorf("OperatorClientID not configured") logger.Error(err, "SPIFFE auth requires OperatorClientID") @@ -281,36 +281,40 @@ func (r *ClientRegistrationReconciler) reconcileOne( return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } - jwtSVIDPath := r.JWTSVIDPath - if jwtSVIDPath == "" { - jwtSVIDPath = "/opt/jwt_svid.token" + socket := r.SpiffeSocket + if socket == "" { + socket = "unix:///spiffe-workload-api/spire-agent.sock" } - // Path traversal protection: only allow reading from designated directories - cleanPath := filepath.Clean(jwtSVIDPath) - if !strings.HasPrefix(cleanPath, "/opt/") && !strings.HasPrefix(cleanPath, "/var/run/secrets/") { - err := fmt.Errorf("JWT-SVID path %q outside allowed directories (/opt/, /var/run/secrets/)", jwtSVIDPath) - logger.Error(err, "invalid JWT-SVID path") + // Fetch a JWT-SVID directly from the SPIRE workload API using the go-spiffe SDK. + // This is the same socket used by verifiedFetch — no sidecar or file needed. + // The audience must be the Keycloak realm's issuer URL (external URL, not in-cluster). + // The JWT-SVID audience must be the Keycloak realm's issuer URL (external/public URL). + // Keycloak's FederatedJWTClientValidator checks aud == its own issuer string. + audience := fmt.Sprintf("%s/realms/%s", strings.TrimRight(ab.KeycloakURL, "/"), ab.KeycloakRealm) + spiffeClient, err := workloadapi.New(ctx, workloadapi.WithAddr(socket)) + if err != nil { + logger.Error(err, "failed to create SPIFFE workload API client") if r.Recorder != nil { - r.Recorder.Eventf(owner, corev1.EventTypeWarning, "InvalidJWTSVIDPath", - "JWT-SVID path %q rejected: must be under /opt/ or /var/run/secrets/", jwtSVIDPath) + r.Recorder.Eventf(owner, corev1.EventTypeWarning, "SPIFFEClientFailed", + "Failed to connect to SPIRE workload API at %s: %v", socket, err) } - return ctrl.Result{}, err // fail permanently on config error + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } + defer spiffeClient.Close() - jwtSVID, err := os.ReadFile(cleanPath) + svid, err := spiffeClient.FetchJWTSVID(ctx, jwtsvid.Params{Audience: audience}) if err != nil { - logger.Error(err, "read JWT-SVID failed", "path", cleanPath) + logger.Error(err, "failed to fetch JWT-SVID from SPIRE") if r.Recorder != nil { - r.Recorder.Eventf(owner, corev1.EventTypeWarning, "JWTSVIDReadFailed", - "Failed to read JWT-SVID from %s: %v. Check spiffe-helper sidecar configuration.", cleanPath, err) + r.Recorder.Event(owner, corev1.EventTypeWarning, "JWTSVIDFetchFailed", + "Failed to fetch JWT-SVID from SPIRE workload API. Check SPIRE is running and the operator has a registered identity.") } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } - // WARNING: JWT-SVID is a bearer token - must never appear in logs or error messages - // to prevent token exposure. All code paths must handle jwtSVID as sensitive data. - token, err = kc.JWTSVIDGrantToken(ctx, ab.KeycloakRealm, r.OperatorClientID, string(jwtSVID)) + // WARNING: JWT-SVID is a bearer token — must never appear in logs or error messages. + token, err = kc.JWTSVIDGrantToken(ctx, ab.KeycloakRealm, r.OperatorClientID, svid.Marshal()) if err != nil { logger.Error(err, "Keycloak JWT-SVID authentication failed") if r.Recorder != nil { @@ -319,7 +323,7 @@ func (r *ClientRegistrationReconciler) reconcileOne( } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } - logger.V(1).Info("authenticated with JWT-SVID", "clientId", r.OperatorClientID) + logger.V(1).Info("authenticated with JWT-SVID via SPIRE workload API", "clientId", r.OperatorClientID) } else { // Admin credentials path (legacy) adminUser, adminPass, err := r.resolveKeycloakAdminCredentials(ctx)