feat(bastion): add cluster-wide SSH bastion as Terraform

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ repos:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
-        exclude: ^applications/.*/internal/
+        exclude: ^(applications/.*/internal/|base-infrastructure/charts/)
       - id: check-case-conflict
       - id: detect-private-key
 

base-infrastructure/charts/ssh-bastion/.helmignore

@@ -0,0 +1,7 @@
+# Patterns to ignore when building packages.
+.DS_Store
+.git/
+.gitignore
+*.tmproj
+# helm-unittest suites — not part of the deployed chart
+tests/

base-infrastructure/charts/ssh-bastion/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: ssh-bastion
+description: Cluster-wide SSH bastion / jump host (port-forwarding + ProxyJump), applied by Terraform.
+type: application
+version: 0.1.0

base-infrastructure/charts/ssh-bastion/files/assert-sshd.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+# Fail closed. The linuxserver image boots sshd even when our drop-in is ignored (wrong
+# mount path, an image change to the include behaviour, etc.), silently falling back to
+# insecure defaults. This asserts sshd's *effective* config (`sshd -T`) contains every
+# directive from the mounted drop-in; on mismatch it exits non-zero so Kubernetes pulls
+# the pod from the LoadBalancer (readiness) and restarts / crash-loops it (startup +
+# liveness) instead of accepting connections with unintended settings.
+#
+# Expected directives are read from the drop-in itself, so this can never drift from the
+# config we ship.
+DROPIN=/config/sshd/sshd_config.d/100-ifrc-forwarding.conf
+PORT=2222
+
+# sshd must be accepting connections ...
+nc -z 127.0.0.1 "$PORT" || exit 1
+
+# ... pass the host keys explicitly (as the image's service does) so `sshd -T` works even
+# when this probe runs as a non-root user (otherwise it exits "no hostkeys available").
+h=""
+for k in /config/ssh_host_keys/ssh_host_*_key; do
+  [ -f "$k" ] && h="$h -h $k"
+done
+
+eff=$(sshd.pam -T -f /config/sshd/sshd_config $h 2>/dev/null | tr 'A-Z' 'a-z')
+
+missing=$(grep -vE '^[[:space:]]*(#|$)' "$DROPIN" | tr 'A-Z' 'a-z' | while IFS= read -r line; do
+  d=$(printf '%s' "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+  [ -n "$d" ] || continue
+  printf '%s\n' "$eff" | grep -qF "$d" || printf '%s\n' "$d"
+done)
+
+if [ -n "$missing" ]; then
+  echo "sshd config assertion FAILED (missing directives):" >&2
+  printf '%s\n' "$missing" >&2
+  exit 1
+fi

base-infrastructure/charts/ssh-bastion/files/sshd_config.d/100-ifrc-forwarding.conf

@@ -0,0 +1,38 @@
+# Jump host for port-forwarding / ProxyJump. Key-only auth (PasswordAuthentication
+# no, GatewayPorts no, X11Forwarding no are already set by the image defaults).
+AllowTcpForwarding yes
+
+# Agent forwarding is a security downgrade (a compromised host can hijack the connecting
+# user's agent) and is unnecessary for a port-forward/ProxyJump host — disable it
+# explicitly rather than relying on the default.
+AllowAgentForwarding no
+{{- if .Values.forwardingOnly }}
+
+# Forwarding-only: no interactive shell, no command exec, no SFTP/SCP. ForceCommand also
+# overrides subsystem requests, so sftp is blocked too. Only -N / -L / -D / -J / -W work.
+# This is what prevents a key holder from reading the box (e.g. the host private key).
+PermitTTY no
+ForceCommand /bin/false
+{{- end }}
+{{- with .Values.sshd.permitOpen }}
+
+# Allowlist of permitted forward destinations (narrows how far a key can pivot).
+PermitOpen {{ join " " . }}
+{{- end }}
+
+# Authoritative, declarative authorized_keys (mounted read-only). A single fixed file
+# instead of the image's PUBLIC_KEY_DIR (which only ever *appends* to a persistent file)
+# so that removing a key actually revokes access.
+AuthorizedKeysFile /etc/ssh/authorized_keys
+
+# Auth hardening (internet-exposed LoadBalancer)
+PermitRootLogin no
+KbdInteractiveAuthentication no
+MaxAuthTries 3
+LoginGraceTime 30
+AllowUsers {{ .Values.user }}
+
+# Audit trail (log key fingerprint per login) + reap dead sessions/tunnels
+LogLevel VERBOSE
+ClientAliveInterval 300
+ClientAliveCountMax 2

base-infrastructure/charts/ssh-bastion/templates/_helpers.tpl

@@ -0,0 +1,12 @@
+{{- define "ssh-bastion.name" -}}
+ssh-bastion
+{{- end -}}
+
+{{/* Common labels. `app` is also the (immutable) StatefulSet selector, so keep it stable. */}}
+{{- define "ssh-bastion.labels" -}}
+app: ssh-bastion
+app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+{{- with .Values.environment }}
+environment: {{ . | quote }}
+{{- end }}
+{{- end -}}

base-infrastructure/charts/ssh-bastion/templates/configmap-authorized-keys.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "ssh-bastion.name" . }}-authorized-keys
+  labels:
+    {{- include "ssh-bastion.labels" . | nindent 4 }}
+data:
+  # All public keys concatenated into one authoritative authorized_keys file.
+  authorized_keys: |
+    {{- range .Values.keys }}
+    {{ . }}
+    {{- end }}

base-infrastructure/charts/ssh-bastion/templates/configmap-config.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "ssh-bastion.name" . }}-config
+  labels:
+    {{- include "ssh-bastion.labels" . | nindent 4 }}
+data:
+  # sshd drop-in. Must be mounted under /config/sshd/sshd_config.d/ — the image runs
+  # `sshd -f /config/sshd/sshd_config` and only includes drop-ins from that directory; the
+  # stock /etc/ssh/sshd_config.d/ is not read.
+  100-ifrc-forwarding.conf: |
+    {{- tpl (.Files.Get "files/sshd_config.d/100-ifrc-forwarding.conf") . | nindent 4 }}
+  # Fail-closed startup/readiness/liveness assertion (see the script for details).
+  assert-sshd.sh: |
+    {{- .Files.Get "files/assert-sshd.sh" | nindent 4 }}

base-infrastructure/charts/ssh-bastion/templates/networkpolicy.yaml

@@ -0,0 +1,38 @@
+{{- if .Values.networkPolicy.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: {{ include "ssh-bastion.name" . }}
+  labels:
+    {{- include "ssh-bastion.labels" . | nindent 4 }}
+spec:
+  podSelector:
+    matchLabels:
+      app: ssh-bastion
+  policyTypes:
+    - Ingress
+    - Egress
+  ingress:
+    # SSH in from anywhere (access is gated by public-key auth).
+    - ports:
+        - protocol: TCP
+          port: {{ .Values.service.port }}
+  egress:
+    # Cluster DNS.
+    - ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+    # Allowed forward/egress destinations. This is an allowlist, so anything not listed —
+    # the public internet and the cloud metadata endpoint (169.254.169.254) included — is
+    # denied by construction. If you broaden a CIDR to cover link-local, add an
+    # `except: [169.254.0.0/16]` to that ipBlock (it must be a subset of the cidr).
+    {{- with .Values.networkPolicy.allowedEgressCIDRs }}
+    - to:
+        {{- range . }}
+        - ipBlock:
+            cidr: {{ . | quote }}
+        {{- end }}
+    {{- end }}
+{{- end }}

base-infrastructure/charts/ssh-bastion/templates/service.yaml

@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "ssh-bastion.name" . }}
+  labels:
+    {{- include "ssh-bastion.labels" . | nindent 4 }}
+  {{- with .Values.service.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  type: {{ .Values.service.type }}
+  # Open to the internet; access is gated by SSH public-key auth only (team members do not
+  # have static source IPs, so no loadBalancerSourceRanges).
+  {{- if and (eq .Values.service.type "LoadBalancer") .Values.service.loadBalancerIP }}
+  loadBalancerIP: {{ .Values.service.loadBalancerIP | quote }}
+  {{- end }}
+  {{- if ne .Values.service.type "ClusterIP" }}
+  # Preserve the real client source IP (otherwise SNAT'd to a node IP) so the VERBOSE sshd
+  # audit log records who connected.
+  externalTrafficPolicy: Local
+  {{- end }}
+  selector:
+    app: ssh-bastion
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: {{ .Values.service.port }}
+      {{- if and (eq .Values.service.type "NodePort") .Values.service.nodePort }}
+      nodePort: {{ .Values.service.nodePort }}
+      {{- end }}

base-infrastructure/charts/ssh-bastion/templates/statefulset.yaml

@@ -0,0 +1,103 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "ssh-bastion.name" . }}
+  labels:
+    {{- include "ssh-bastion.labels" . | nindent 4 }}
+spec:
+  replicas: 1
+  serviceName: {{ include "ssh-bastion.name" . }}
+  selector:
+    matchLabels:
+      app: ssh-bastion
+  template:
+    metadata:
+      labels:
+        app: ssh-bastion
+      annotations:
+        # Roll the pod when keys or sshd config change (the container ingests both only at
+        # start), otherwise a ConfigMap edit applies but the running pod keeps the old data.
+        checksum/authorized-keys: {{ include (print $.Template.BasePath "/configmap-authorized-keys.yaml") . | sha256sum }}
+        checksum/config: {{ include (print $.Template.BasePath "/configmap-config.yaml") . | sha256sum }}
+    spec:
+      # The bastion never talks to the Kubernetes API — don't mount a cluster credential
+      # onto an internet-exposed jump host.
+      automountServiceAccountToken: false
+      containers:
+        - name: ssh-bastion
+          image: {{ .Values.image | quote }}
+          ports:
+            - containerPort: {{ .Values.service.port }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          env:
+            - name: PUID
+              value: "1000"
+            - name: PGID
+              value: "1000"
+            - name: USER_NAME
+              value: {{ .Values.user | quote }}
+            - name: PASSWORD_ACCESS
+              value: "false"
+            - name: SUDO_ACCESS
+              value: "false"
+          # Gate readiness/liveness until sshd is up and the config assertion passes. If the
+          # config never applies the pod never starts (CrashLoopBackOff) rather than serving.
+          startupProbe:
+            exec:
+              command: ["sh", "/etc/ifrc/assert-sshd.sh"]
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 30
+          # Pull the pod out of the Service/LoadBalancer endpoints if the effective config
+          # ever regresses — no connections are routed to a misconfigured bastion.
+          readinessProbe:
+            exec:
+              command: ["sh", "/etc/ifrc/assert-sshd.sh"]
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 2
+          # Restart (crash-loop) the pod on a runtime config regression.
+          livenessProbe:
+            exec:
+              command: ["sh", "/etc/ifrc/assert-sshd.sh"]
+            periodSeconds: 60
+            timeoutSeconds: 5
+            failureThreshold: 3
+          volumeMounts:
+            # Authoritative authorized_keys — sshd reads it via AuthorizedKeysFile (drop-in).
+            # root-owned read-only file, which satisfies sshd's ownership checks.
+            - name: authorized-keys
+              mountPath: /etc/ssh/authorized_keys
+              subPath: authorized_keys
+              readOnly: true
+            - name: config-volume
+              mountPath: /config
+            # Must live under /config/sshd/sshd_config.d/ for the image to include it.
+            - name: config
+              mountPath: /config/sshd/sshd_config.d/100-ifrc-forwarding.conf
+              subPath: 100-ifrc-forwarding.conf
+              readOnly: true
+            - name: config
+              mountPath: /etc/ifrc/assert-sshd.sh
+              subPath: assert-sshd.sh
+              readOnly: true
+      volumes:
+        - name: authorized-keys
+          configMap:
+            name: {{ include "ssh-bastion.name" . }}-authorized-keys
+        - name: config
+          configMap:
+            name: {{ include "ssh-bastion.name" . }}-config
+  # Persists the server host keys across pod restarts (avoids host-key-changed warnings).
+  volumeClaimTemplates:
+    - metadata:
+        name: config-volume
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        {{- with .Values.persistence.storageClass }}
+        storageClassName: {{ . | quote }}
+        {{- end }}
+        resources:
+          requests:
+            storage: {{ .Values.persistence.size | quote }}