From d54a12c701ff6acd0d5fcb3ae5db082e3ed821da Mon Sep 17 00:00:00 2001 From: Benjamin Pickard Date: Mon, 25 Aug 2025 16:38:57 -0400 Subject: [PATCH 1/3] Manual feature backport of cert rotation for whereabouts Includes cert check script logic and delpoyment of token watcher daemonset in openshift-mutlus namespace to watch for token change Signed-off-by: Benjamin Pickard --- bindata/network/multus/multus.yaml | 276 ++++++++++++++++++++++++++-- pkg/network/multus.go | 9 +- pkg/network/multus_test.go | 2 +- pkg/network/network_metrics_test.go | 2 +- 4 files changed, 263 insertions(+), 26 deletions(-) diff --git a/bindata/network/multus/multus.yaml b/bindata/network/multus/multus.yaml index ba95849004..b4510dd0dd 100644 --- a/bindata/network/multus/multus.yaml +++ b/bindata/network/multus/multus.yaml @@ -106,6 +106,23 @@ data: fi rm -Rf $UPGRADE_DIRECTORY --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: whereabouts-flatfile-config + namespace: openshift-multus +data: + whereabouts.conf: | + { + "datastore": "kubernetes", + "kubernetes": { + "kubeconfig": "/etc/kubernetes/cni/net.d/whereabouts.d/whereabouts.kubeconfig" + }, + "reconciler_cron_expression": "30 4 * * *", + "log_level": "verbose", + "configuration_path": "/etc/kubernetes/cni/net.d/whereabouts.d" + } +--- kind: ConfigMap apiVersion: v1 metadata: @@ -157,7 +174,6 @@ data: "bootstrapKubeconfig": "{{ .KubeletKubeconfigPath }}", "certDir": "/etc/cni/multus/certs", "certDuration": "24h" - }, {{ end }} "cniConfigDir": "/host/etc/cni/net.d", "multusConfigFile": "auto", @@ -326,7 +342,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - terminationGracePeriodSeconds: 10 + terminationGracePeriodSeconds: 30 volumes: - name: system-cni-dir hostPath: @@ -546,21 +562,43 @@ spec: CNI_BIN_DIR=${CNI_BIN_DIR:-"/host/opt/cni/bin/"} WHEREABOUTS_KUBECONFIG_FILE_HOST=${WHEREABOUTS_KUBECONFIG_FILE_HOST:-"/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig"} - CNI_CONF_DIR=${CNI_CONF_DIR:-"/host/etc/cni/net.d"} + CNI_CONF_DIR=${CNI_CONF_DIR:-"/host{{ .SystemCNIConfDir }}"} + WHEREABOUTS_RECONCILER_CRON=${WHEREABOUTS_RECONCILER_CRON:-30 4 * * *} # Make a whereabouts.d directory (for our kubeconfig) mkdir -p $CNI_CONF_DIR/whereabouts.d WHEREABOUTS_KUBECONFIG=$CNI_CONF_DIR/whereabouts.d/whereabouts.kubeconfig - WHEREABOUTS_GLOBALCONFIG=$CNI_CONF_DIR/whereabouts.d/whereabouts.conf + WHEREABOUTS_CONF_FILE=$CNI_CONF_DIR/whereabouts.d/whereabouts.conf + WHEREABOUTS_KUBECONFIG_LITERAL=$(echo "$WHEREABOUTS_KUBECONFIG" | sed -e s'|/host||') + + # Write the nodename to the whereabouts.d directory for standardized hostname reference across cloud providers + echo $NODENAME > $CNI_CONF_DIR/whereabouts.d/nodename - # ------------------------------- Generate a "kube-config" SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount KUBE_CA_FILE=${KUBE_CA_FILE:-$SERVICE_ACCOUNT_PATH/ca.crt} - SERVICEACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token) + SERVICE_ACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token) + SERVICE_ACCOUNT_TOKEN_PATH=$SERVICE_ACCOUNT_PATH/token SKIP_TLS_VERIFY=${SKIP_TLS_VERIFY:-false} + function log() + { + echo "$(date -Iseconds) ${1}" + } + + function error() + { + log "ERR: {$1}" + } + + function warn() + { + log "WARN: {$1}" + } + + + function generateKubeConfig { # Check if we're running as a k8s pod. if [ -f "$SERVICE_ACCOUNT_PATH/token" ]; then # We're running as a k8d pod - expect some variables. @@ -577,6 +615,12 @@ spec: TLS_CFG="certificate-authority-data: $(cat $KUBE_CA_FILE | base64 | tr -d '\n')" fi + # Kubernetes service address must be wrapped if it is IPv6 address + KUBERNETES_SERVICE_HOST_WRAP=$KUBERNETES_SERVICE_HOST + if [ "$KUBERNETES_SERVICE_HOST_WRAP" != "${KUBERNETES_SERVICE_HOST_WRAP#*:[0-9a-fA-F]}" ]; then + KUBERNETES_SERVICE_HOST_WRAP=\[$KUBERNETES_SERVICE_HOST_WRAP\] + fi + # Write a kubeconfig file for the CNI plugin. Do this # to skip TLS verification for now. We should eventually support # writing more complete kubeconfig files. This is only used @@ -584,18 +628,18 @@ spec: touch $WHEREABOUTS_KUBECONFIG chmod ${KUBECONFIG_MODE:-600} $WHEREABOUTS_KUBECONFIG cat > $WHEREABOUTS_KUBECONFIG < $WHEREABOUTS_GLOBALCONFIG < $WHEREABOUTS_CONF_FILE < $WHEREABOUTS_KUBECONFIG < Date: Wed, 18 Jun 2025 16:10:45 +0200 Subject: [PATCH 2/3] Add release version annotation to whereabouts-controller and whereabouts-token-watcher --- bindata/network/multus/multus.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindata/network/multus/multus.yaml b/bindata/network/multus/multus.yaml index b4510dd0dd..7ae9373d36 100644 --- a/bindata/network/multus/multus.yaml +++ b/bindata/network/multus/multus.yaml @@ -859,7 +859,7 @@ metadata: kubernetes.io/description: | This deamon watches over the whereabouts service account token and CA file for changes and will regenerate a kubeconfig if changes are seen - relase.openshift.io.version: "{{.ReleaseVersion}}" + release.openshift.io/version: "{{.ReleaseVersion}}" spec: selector: matchLabels: From 28d9a45fdbd783a8268d4d0a9a2911c453946134 Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Tue, 18 Nov 2025 02:32:01 -0500 Subject: [PATCH 3/3] Fix: whereabouts-token-watcher DaemonSet improvements This commit introduces several improvements to the whereabouts-token-watcher DaemonSet: - Corrects a typo in the description. - Adds logging helper functions for better script output. - Ensures the service account token is read dynamically during kubeconfig generation. - Fixes the logical grouping in the CA file change detection condition. - Increases the kubeconfig regeneration check interval to 60 seconds. - Mounts the host's CNI network directory for proper CNI configuration management. Signed-off-by: Peng Liu Signed-off-by: Benjamin Pickard --- bindata/network/multus/multus.yaml | 28 +++++++++++++++++++--------- pkg/network/multus_test.go | 2 +- pkg/network/network_metrics_test.go | 2 +- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/bindata/network/multus/multus.yaml b/bindata/network/multus/multus.yaml index 7ae9373d36..ed2d6c855d 100644 --- a/bindata/network/multus/multus.yaml +++ b/bindata/network/multus/multus.yaml @@ -857,7 +857,7 @@ metadata: namespace: openshift-multus annotations: kubernetes.io/description: | - This deamon watches over the whereabouts service account token and CA + This daemon watches over the whereabouts service account token and CA file for changes and will regenerate a kubeconfig if changes are seen release.openshift.io/version: "{{.ReleaseVersion}}" spec: @@ -877,6 +877,7 @@ spec: labels: app: whereabouts-token-watcher spec: + hostNetwork: true nodeSelector: kubernetes.io/os: linux priorityClassName: "system-node-critical" @@ -894,11 +895,16 @@ spec: set -u -e + # Helper functions for logging + function log { echo "$(date -Iseconds) $@"; } + function warn { echo "$(date -Iseconds) WARNING: $@" >&2; } + function error { echo "$(date -Iseconds) ERROR: $@" >&2; } + SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount KUBE_CA_FILE=${KUBE_CA_FILE:-$SERVICE_ACCOUNT_PATH/ca.crt} - SERVICE_ACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token) SERVICE_ACCOUNT_TOKEN_PATH=$SERVICE_ACCOUNT_PATH/token SKIP_TLS_VERIFY=${SKIP_TLS_VERIFY:-false} + WHEREABOUTS_KUBECONFIG=${CNI_CONF_DIR:-/host/etc/cni/net.d}/whereabouts.d/whereabouts.kubeconfig function generateKubeConfig { # Check if we're running as a k8s pod. @@ -940,7 +946,7 @@ spec: users: - name: whereabouts user: - token: "${SERVICE_ACCOUNT_TOKEN}" + token: "$(cat $SERVICE_ACCOUNT_TOKEN_PATH)" contexts: - name: whereabouts-context context: @@ -971,20 +977,20 @@ spec: export LAST_SERVICEACCOUNT_MD5SUM="$(get_token_md5sum)" export LAST_KUBE_CA_FILE_MD5SUM="$(get_ca_file_md5sum)" - echo "Sleep and Watching for service account token and CA file changes..." + log "Sleep and Watching for service account token and CA file changes..." # enter sleep/watch loop while true; do # Check the md5sum of the service account token and ca. svcaccountsum="$(get_token_md5sum)" casum="$(get_ca_file_md5sum)" - if [ "$svcaccountsum" != "$LAST_SERVICEACCOUNT_MD5SUM" ] || ! [ "$SKIP_TLS_VERIFY" == "true" ] && [ "$casum" != "$LAST_KUBE_CA_FILE_MD5SUM" ]; then + if [ "$svcaccountsum" != "$LAST_SERVICEACCOUNT_MD5SUM" ] || ( [ "$SKIP_TLS_VERIFY" != "true" ] && [ "$casum" != "$LAST_KUBE_CA_FILE_MD5SUM" ] ); then log "Detected service account or CA file change, regenerating kubeconfig..." generateKubeConfig LAST_SERVICEACCOUNT_MD5SUM="$svcaccountsum" LAST_KUBE_CA_FILE_MD5SUM="$casum" fi - sleep 1s + sleep 60s done resources: @@ -994,7 +1000,9 @@ spec: terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: whereabouts-flatfile-configmap - mountPath: /etc/wherabouts/config + mountPath: /etc/whereabouts/config + - mountPath: /host/etc/cni/net.d + name: cni-net-dir env: - name: KUBERNETES_SERVICE_PORT value: "{{.KUBERNETES_SERVICE_PORT}}" @@ -1004,13 +1012,15 @@ spec: value: "/host/opt/cni/bin/" - name: CNI_CONF_DIR value: "/host/etc/cni/net.d" - - name: SLEEP - value: "false" - name: WHEREABOUTS_NAMESPACE value: "openshift-multus" volumes: - name: whereabouts-flatfile-configmap configMap: name: whereabouts-flatfile-config + - name: cni-net-dir + hostPath: + path: {{ .SystemCNIConfDir }} + type: Directory {{- end}} --- diff --git a/pkg/network/multus_test.go b/pkg/network/multus_test.go index b5de9f5ef8..3545d8fe94 100644 --- a/pkg/network/multus_test.go +++ b/pkg/network/multus_test.go @@ -49,7 +49,7 @@ func TestRenderMultus(t *testing.T) { g.Expect(objs).To(ContainElement(HaveKubernetesID("DaemonSet", "openshift-multus", "multus"))) // It's important that the namespace is first - g.Expect(len(objs)).To(Equal(32), "Expected 32 multus related objects") + g.Expect(len(objs)).To(Equal(31), "Expected 31 multus related objects") g.Expect(objs[0]).To(HaveKubernetesID("CustomResourceDefinition", "", "network-attachment-definitions.k8s.cni.cncf.io")) g.Expect(objs).To(ContainElement(HaveKubernetesID("Namespace", "", "openshift-multus"))) g.Expect(objs).To(ContainElement(HaveKubernetesID("ServiceAccount", "openshift-multus", "multus-ancillary-tools"))) diff --git a/pkg/network/network_metrics_test.go b/pkg/network/network_metrics_test.go index acf1220c07..c94a38df70 100644 --- a/pkg/network/network_metrics_test.go +++ b/pkg/network/network_metrics_test.go @@ -49,7 +49,7 @@ func TestRenderNetworkMetricsDaemon(t *testing.T) { // Check rendered object - g.Expect(len(objs)).To(Equal(32), "Expected 32 multus related objects") + g.Expect(len(objs)).To(Equal(31), "Expected 31 multus related objects") g.Expect(objs).To(ContainElement(HaveKubernetesID("DaemonSet", "openshift-multus", "network-metrics-daemon"))) g.Expect(objs).To(ContainElement(HaveKubernetesID("Service", "openshift-multus", "network-metrics-service"))) g.Expect(objs).To(ContainElement(HaveKubernetesID("ClusterRole", "", "metrics-daemon-role")))