diff --git a/.github/scripts/deploy-es.sh b/.github/scripts/deploy-es.sh new file mode 100755 index 0000000..2b13c62 --- /dev/null +++ b/.github/scripts/deploy-es.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Story 16.5 — deploy single-node Elasticsearch container(s) into a kind cluster, one per +# federation sidecar, reachable at the in-cluster Service name the install job overrides each +# sidecars[].elasticsearch.url to. Used by .github/workflows/federation-helm.yml. +# +# Usage: deploy-es.sh +# topology : single-cluster | three-region | heterogeneous-ready | es-version | secrets +# es-version-list : comma-separated ES major versions, e.g. "8" or "8,8,9" +# +# Service names (must match the --set sidecars[N].elasticsearch.url=... in the install jobs): +# * 1-ES topologies (single-cluster / es-version / secrets) -> Service "es" +# * 3-ES topologies (three-region / heterogeneous-ready) -> "es-us-east-1", "es-eu-west-1", +# "es-ap-south-1" (order = list order) +# +# Each ES runs single-node, security disabled, small heap (fits the 7 GB GitHub runner). +# `vm.max_map_count=262144` MUST already be set on the runner (the workflow does this). +set -euo pipefail + +TOPOLOGY="${1:?usage: deploy-es.sh }" +ES_LIST="${2:?usage: deploy-es.sh }" + +# ES major version -> full image coordinate (CLAUDE.md ES Version Matrix). +es_image() { + case "$1" in + 6) echo "docker.elastic.co/elasticsearch/elasticsearch:6.8.23" ;; + 7) echo "docker.elastic.co/elasticsearch/elasticsearch:7.17.29" ;; + 8) echo "docker.elastic.co/elasticsearch/elasticsearch:8.18.3" ;; + 9) echo "docker.elastic.co/elasticsearch/elasticsearch:9.0.3" ;; + *) echo "unsupported ES version: $1" >&2; exit 1 ;; + esac +} + +# Resolve the Service names for this topology, positionally aligned with the ES list. +case "$TOPOLOGY" in + three-region|heterogeneous-ready) + NAMES=(es-us-east-1 es-eu-west-1 es-ap-south-1) + ;; + single-cluster|es-version|secrets) + NAMES=(es) + ;; + *) + echo "unsupported topology: $TOPOLOGY" >&2; exit 1 ;; +esac + +# Split the comma-separated version list into an array. +IFS=',' read -r -a VERSIONS <<< "$ES_LIST" + +if [ "${#VERSIONS[@]}" -gt "${#NAMES[@]}" ]; then + echo "::error::deploy-es.sh: ${#VERSIONS[@]} ES versions but only ${#NAMES[@]} service name(s) for topology $TOPOLOGY" >&2 + exit 1 +fi + +deploy_one() { + local name="$1" version="$2" image + image="$(es_image "$version")" + echo "Deploying Elasticsearch ${version} as Service '${name}' (${image})" + kubectl apply -f - < +# secret-name : the name of the Secret the controller will materialize (chart references this +# via sidecars[0].elasticsearch.credentialsSecretName), e.g. "es-creds". +# +# Why re-seal in CI (project_sealed_secrets_review_gotchas): +# * The committed examples/sealed-secrets/*.yaml are NON-FUNCTIONAL placeholders — a SealedSecret +# is decryptable ONLY by the specific controller instance that issued the cert. We therefore +# create a fresh plaintext Secret, `kubeseal` it with the live cluster's cert, and apply. +# * The kubeseal CLI version is pinned to the controller chart appVersion (NOT the chart version). +set -euo pipefail + +SECRET_NAME="${1:?usage: install-sealed-secrets.sh }" + +# Pin the controller chart + kubeseal CLI so the CLI/controller versions match (gotcha: +# kubeseal CLI version must equal the controller appVersion, not the Helm chart version). +SEALED_SECRETS_CHART_VERSION="2.16.2" # helm chart version +KUBESEAL_VERSION="0.27.2" # == controller appVersion of chart 2.16.2 + +# 1. Install the controller (repo moved bitnami-labs -> bitnami.github.io). +helm repo add sealed-secrets https://bitnami-labs.github.io/sealed-secrets >/dev/null 2>&1 || \ + helm repo add sealed-secrets https://bitnami.github.io/sealed-secrets +helm repo update sealed-secrets +helm upgrade --install sealed-secrets sealed-secrets/sealed-secrets \ + --namespace kube-system \ + --version "${SEALED_SECRETS_CHART_VERSION}" \ + --set fullnameOverride=sealed-secrets-controller \ + --wait --timeout 180s + +# 2. Install the matching kubeseal CLI. +curl -sSL "https://github.com/bitnami-labs/sealed-secrets/releases/download/v${KUBESEAL_VERSION}/kubeseal-${KUBESEAL_VERSION}-linux-amd64.tar.gz" \ + | tar -xz kubeseal +sudo install -m 0755 kubeseal /usr/local/bin/kubeseal +rm -f kubeseal + +# 3. Wait for the controller to be Ready, then fetch its public cert. +kubectl rollout status deployment/sealed-secrets-controller -n kube-system --timeout=180s +kubeseal --controller-name=sealed-secrets-controller --controller-namespace=kube-system \ + --fetch-cert > /tmp/sealed-secrets-cert.pem + +# 4. Build a plaintext ES-auth Secret (matching the 16.3 es-* key contract), seal it with the +# live cert, and apply the SealedSecret (the plaintext Secret is NEVER applied to the cluster). +kubectl create secret generic "${SECRET_NAME}" \ + --dry-run=client -o yaml \ + --from-literal=es-auth-method=basic \ + --from-literal=es-username=elastic \ + --from-literal=es-password=changeme \ + | kubeseal --cert /tmp/sealed-secrets-cert.pem --format yaml \ + | kubectl apply -f - + +# 5. Wait for the controller to materialize the real Secret from the SealedSecret. +for _ in $(seq 1 30); do + if kubectl get secret "${SECRET_NAME}" >/dev/null 2>&1; then + echo "SealedSecrets materialized Secret '${SECRET_NAME}'." + exit 0 + fi + sleep 2 +done + +echo "::error::SealedSecrets controller did not materialize Secret '${SECRET_NAME}' in time" >&2 +kubectl get sealedsecret,secret 2>/dev/null || true +exit 1 diff --git a/.github/workflows/federation-helm.yml b/.github/workflows/federation-helm.yml new file mode 100644 index 0000000..1c39bff --- /dev/null +++ b/.github/workflows/federation-helm.yml @@ -0,0 +1,519 @@ +# Comprehensive CI for the softclient4es-federation Helm chart (Epic 16, Story 16.5). +# Validates: lint + template + kubeconform + golden-diff (static, hard blocker) AND +# per-topology / per-ES-version / secret-backend / upgrade / uninstall installs on +# ephemeral kind clusters. +# +# This is the dedicated CHART repo (softclient4es-helm) — it is chart-only and CANNOT +# build images via sbt. CI therefore NEVER builds: the live-install jobs `docker pull` +# the PUBLIC DockerHub federation + sidecar images and `kind load` them. checkout@v4, +# `sudo sysctl -w vm.max_map_count=262144` before any Elasticsearch boots. +# +# IMAGE TAGS: the chart's image refs default to the Chart.yaml appVersion. The live jobs +# pull the public DockerHub tags (federation image.tag:"" -> appVersion; sidecar tag -> +# appVersion) and `kind load` them — no per-image --set override, so the federation + +# sidecar tags stay consistent. The committed goldens are appVersion renders, so the +# golden gate renders with NO --set image.tag. +# +# IMAGE AVAILABILITY (OQ-1): neither the federation image nor the sidecar tags are +# published on DockerHub yet. The `image-availability` job probes each with +# `docker manifest inspect`; if ANY required image is absent every live-install job is +# SKIPPED WITH A `::warning::` ANNOTATION (not failed). All static-validation gates run +# UNCONDITIONALLY on every PR with zero external deps. +# +# LICENSE: the multi-sidecar (Pro) tiers are BEST-EFFORT, gated on TWO repo secrets +# together (skip-not-fail if either missing), on top of image availability: (1) secret +# SC4ES_PRO_TEST_JWT, (2) secret SC4ES_TEST_PUBLIC_KEY (injected as +# SOFTCLIENT4ES_LICENSE_PUBLIC_KEY so the verifier resolves the test kid OFFLINE). The +# published federation image must itself be Pro-capable (JWT-verifying SPI on classpath, +# Story 16.1 OQ-5) for the Pro JWT to verify — an OSS-only image falls back to Community. +# single-cluster + per-ES-version + secret backends + uninstall + ALL static gates are +# license-free and run unconditionally (when the images are published). +name: Federation Helm Chart + +on: + workflow_dispatch: + pull_request: + paths: + - 'softclient4es-federation/**' + - '.github/workflows/federation-helm.yml' + - '.github/scripts/deploy-es.sh' + - '.github/scripts/install-sealed-secrets.sh' + push: + branches: ['!main'] + paths: + - 'softclient4es-federation/**' + - '.github/workflows/federation-helm.yml' + - '.github/scripts/deploy-es.sh' + - '.github/scripts/install-sealed-secrets.sh' + # Heavy matrix (per-ES-version + secret backends) can be moved here if PR duration > 30 min + # (see the sharding fallback in the chart README). PRs would then run a reduced matrix. + # schedule: + # - cron: '0 3 * * *' + +permissions: + contents: read + +env: + CHART_DIR: softclient4es-federation + HELM_VERSION: v3.16.3 + KUBECONFORM_VERSION: v0.6.7 + KIND_VERSION: v0.24.0 # kind node images current to K8s 1.31 + KIND_NODE_IMAGE: kindest/node:v1.31.0 + # The chart's image refs default to the Chart.yaml appVersion. The live jobs pull the + # public DockerHub tags at this version so the chart DEFAULTS resolve the kind-loaded images. + IMAGE_TAG: '0.2.0' + FED_IMAGE: softnetwork/softclient4es-federation + SIDECAR_IMAGE_PREFIX: softnetwork/softclient4es # softclient4es-arrow-flight-sql + # A pre-baked ADBC client image avoids the smoke Job's runtime pip install (air-gap). When + # empty the chart smoke Pod falls back to python:3.12-slim + runtime pip (PyPI reachable on + # GitHub-hosted runners). PROPOSED image — see OQ-2 / chart README. + TEST_ADBC_IMAGE: '' + +jobs: + # ── 1. STATIC CHECKS — no cluster, unconditional, hard blocker ───────────── + static-checks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Helm + uses: azure/setup-helm@v4 + with: + version: ${{ env.HELM_VERSION }} + - name: Install kubeconform + run: | + curl -sSL "https://github.com/yannh/kubeconform/releases/download/${KUBECONFORM_VERSION}/kubeconform-linux-amd64.tar.gz" \ + | sudo tar -xz -C /usr/local/bin kubeconform + - name: helm lint (chart + all examples) + run: | + helm lint "${CHART_DIR}" + for ex in single-cluster three-region heterogeneous-ready; do + f="${CHART_DIR}/examples/${ex}/values.yaml" + if [ -f "$f" ]; then + helm lint "${CHART_DIR}" -f "$f" + else + echo "::warning::missing example $f (Story 16.4 not merged yet — skipping lint -f)" + fi + done + - name: helm template + kubeconform (base + each example) + run: | + render() { helm template fed "${CHART_DIR}" "$@"; } + # base (0-sidecar) always renders + render | kubeconform -strict -summary -kubernetes-version 1.29.0 -ignore-missing-schemas + for ex in single-cluster three-region heterogeneous-ready; do + f="${CHART_DIR}/examples/${ex}/values.yaml" + if [ -f "$f" ]; then + render -f "$f" | kubeconform -strict -summary -kubernetes-version 1.29.0 -ignore-missing-schemas + else + echo "::warning::missing example $f — skipping kubeconform for $ex" + fi + done + - name: golden-file diff (template drift gate) + run: | + fail=0 + # default (16.1) + two-sidecars (16.2) + secret-auth/ingress-tls (16.3) goldens. + # All committed goldens are appVersion renders — render WITH NO --set image.tag. + helm template fed "${CHART_DIR}" \ + | diff -u "${CHART_DIR}/tests/golden/default.yaml" - || { echo "::error::default.yaml golden drift"; fail=1; } + helm template fed "${CHART_DIR}" -f "${CHART_DIR}/tests/values/two-sidecars.yaml" \ + | diff -u "${CHART_DIR}/tests/golden/two-sidecars.yaml" - || { echo "::error::two-sidecars.yaml golden drift"; fail=1; } + helm template fed "${CHART_DIR}" -f "${CHART_DIR}/tests/values/secret-auth.yaml" \ + | diff -u "${CHART_DIR}/tests/golden/secret-auth.yaml" - || { echo "::error::secret-auth.yaml golden drift"; fail=1; } + helm template fed "${CHART_DIR}" -f "${CHART_DIR}/tests/values/ingress-tls.yaml" \ + | diff -u "${CHART_DIR}/tests/golden/ingress-tls.yaml" - || { echo "::error::ingress-tls.yaml golden drift"; fail=1; } + # Per-example goldens (Story 16.4 committed them at tests/golden/example-.yaml). + for ex in single-cluster three-region heterogeneous-ready; do + f="${CHART_DIR}/examples/${ex}/values.yaml" + g="${CHART_DIR}/tests/golden/example-${ex}.yaml" + if [ -f "$f" ] && [ -f "$g" ]; then + helm template fed "${CHART_DIR}" -f "$f" \ + | diff -u "$g" - || { echo "::error::example-${ex}.yaml golden drift"; fail=1; } + else + echo "::warning::missing example or golden for $ex (16.4 not merged) — skipping golden diff" + fi + done + exit $fail + - name: assert chart renders ZERO kind:Secret (16.3 contract) + run: | + n=$(helm template fed "${CHART_DIR}" | grep -c '^kind: Secret' || true) + [ "$n" = "0" ] || { echo "::error::chart must NOT render a kind:Secret (got $n)"; exit 1; } + - name: heterogeneous-ready discriminator (16.4 A2b — only signal vs three-region) + run: | + # three-region and heterogeneous-ready goldens are BYTE-IDENTICAL by design (Helm + # strips the R2b duckdb-attach comment block). The ONLY signal distinguishing them is + # the commented duckdb-attach preview in the SOURCE values.yaml. Guard against a + # regression that copies one example over the other (passes golden + smoke undetected). + het="${CHART_DIR}/examples/heterogeneous-ready/values.yaml" + tr="${CHART_DIR}/examples/three-region/values.yaml" + if [ -f "$het" ] && [ -f "$tr" ]; then + nh=$(grep -c 'type = "duckdb-attach"' "$het" || true) + nt=$(grep -c 'type = "duckdb-attach"' "$tr" || true) + [ "$nh" = "3" ] || { echo "::error::heterogeneous-ready must keep 3 commented duckdb-attach previews (got $nh)"; exit 1; } + [ "$nt" = "0" ] || { echo "::error::three-region must have 0 duckdb-attach entries (got $nt)"; exit 1; } + else + echo "::warning::examples missing (16.4 not merged) — skipping A2b discriminator" + fi + + # ── 2. IMAGE AVAILABILITY — probe the PUBLIC DockerHub images, prerequisite for installs ─ + # This chart repo cannot build images (no sbt). The federation image + the four sidecar tags + # are pulled from public DockerHub. They are NOT yet published (OQ-1), so probe each tag with + # `docker manifest inspect`; if ANY is absent, the live-install jobs SKIP WITH A WARNING. + image-availability: + runs-on: ubuntu-latest + outputs: + images_published: ${{ steps.probe.outputs.images_published }} # gate for ALL live-install jobs + steps: + - name: Probe public DockerHub images (federation + 4 sidecars) + id: probe + run: | + published=true + probe() { + if docker manifest inspect "$1" >/dev/null 2>&1; then + echo "found: $1" + else + echo "::warning::image not published on DockerHub: $1 — live-install jobs will be SKIPPED (OQ-1)" + published=false + fi + } + probe "docker.io/${FED_IMAGE}:${IMAGE_TAG}" + for v in 6 7 8 9; do + probe "docker.io/${SIDECAR_IMAGE_PREFIX}${v}-arrow-flight-sql:${IMAGE_TAG}" + done + if [ "$published" != "true" ]; then + echo "::warning::one or more federation/sidecar images are not yet published (OQ-1) — skipping all live kind-install jobs; static gates still ran" + fi + echo "images_published=$published" >> "$GITHUB_OUTPUT" + + # ── 3. PER-TOPOLOGY INSTALL — single-cluster (license-free) + 3-cluster (Pro) ─ + install-topology: + runs-on: ubuntu-latest + needs: [static-checks, image-availability] + if: ${{ needs.image-availability.outputs.images_published == 'true' }} + strategy: + fail-fast: false + matrix: + include: + - example: single-cluster + es: '8' # one ES8, hostname `es` + expect_catalogs: 1 + license: false # 1 sidecar -> Community, no license (FACT C) + - example: three-region + es: '8,8,9' # two ES8 + one ES9 (mixed-version) + expect_catalogs: 3 + license: true # 3 sidecars -> Pro JWT required (FACT C/F) + - example: heterogeneous-ready + es: '8,8,9' # 3 ES (R2b placeholders inactive in R1) + expect_catalogs: 3 + license: true + steps: + - uses: actions/checkout@v4 + - name: Set vm.max_map_count (Elasticsearch requirement) + run: sudo sysctl -w vm.max_map_count=262144 + - name: Skip if example missing (Story 16.4 not merged) + id: guard + run: | + f="${CHART_DIR}/examples/${{ matrix.example }}/values.yaml" + if [ ! -f "$f" ]; then + echo "::warning::$f missing — Story 16.4 not merged; skipping install" + echo "skip=true" >> "$GITHUB_OUTPUT" + fi + - name: Skip multi-sidecar install when no Pro test license (FACT C + FACT F, best-effort tier) + id: lic + if: ${{ matrix.license && steps.guard.outputs.skip != 'true' }} + # The 3-sidecar tier needs BOTH: a Pro test JWT and its public verification key (so the + # verifier resolves the test kid OFFLINE). Missing EITHER → SKIP (annotation), never fail. + # (The published federation image must itself be Pro-capable — 16.1 OQ-5 — for the JWT to + # verify; an OSS-only image silently falls back to Community and CrashLoops at 3 clusters.) + run: | + if [ -z "${{ secrets.SC4ES_PRO_TEST_JWT }}" ] || [ -z "${{ secrets.SC4ES_TEST_PUBLIC_KEY }}" ]; then + echo "::warning::missing SC4ES_PRO_TEST_JWT and/or SC4ES_TEST_PUBLIC_KEY — ${{ matrix.example }} (3 sidecars) needs a Pro JWT + its public key (FACT F #2); skipping (best-effort)" + echo "skip=true" >> "$GITHUB_OUTPUT" + fi + - name: Set up Helm + if: ${{ steps.guard.outputs.skip != 'true' && steps.lic.outputs.skip != 'true' }} + uses: azure/setup-helm@v4 + with: { version: "${{ env.HELM_VERSION }}" } + - name: Create kind cluster + if: ${{ steps.guard.outputs.skip != 'true' && steps.lic.outputs.skip != 'true' }} + uses: helm/kind-action@v1.10.0 + with: + version: ${{ env.KIND_VERSION }} + node_image: ${{ env.KIND_NODE_IMAGE }} + cluster_name: fed-${{ matrix.example }} + - name: Pull public images + kind load (federation + sidecars) + if: ${{ steps.guard.outputs.skip != 'true' && steps.lic.outputs.skip != 'true' }} + run: | + docker pull "docker.io/${FED_IMAGE}:${IMAGE_TAG}" + kind load docker-image "docker.io/${FED_IMAGE}:${IMAGE_TAG}" --name "fed-${{ matrix.example }}" + for v in $(echo '${{ matrix.es }}' | tr ',' '\n' | sort -u); do + img="docker.io/${SIDECAR_IMAGE_PREFIX}${v}-arrow-flight-sql:${IMAGE_TAG}" + docker pull "$img" + kind load docker-image "$img" --name "fed-${{ matrix.example }}" + done + - name: Deploy ES container(s) per sidecar (FACT D — reachable ES required) + if: ${{ steps.guard.outputs.skip != 'true' && steps.lic.outputs.skip != 'true' }} + run: ./.github/scripts/deploy-es.sh "${{ matrix.example }}" '${{ matrix.es }}' + - name: Create Pro license + public-key Secret (multi-sidecar only — FACT F #2) + if: ${{ matrix.license && steps.guard.outputs.skip != 'true' && steps.lic.outputs.skip != 'true' }} + run: | + # ONE Secret carries both the JWT (license-key) and the public verification JWK + # (license-public-key → SOFTCLIENT4ES_LICENSE_PUBLIC_KEY, the air-gap path in + # LicenseKeyVerifier.loadPublicKey Step 3 — without it the test kid never resolves). + kubectl create secret generic sc4es-license \ + --from-literal=license-key="${{ secrets.SC4ES_PRO_TEST_JWT }}" \ + --from-literal=license-public-key="${{ secrets.SC4ES_TEST_PUBLIC_KEY }}" + # The three-region example pins per-region license.secretName=sc4es-pro-license; also + # create that name so its bearer-auth sidecars + the federation share the test token. + kubectl create secret generic sc4es-pro-license \ + --from-literal=license-key="${{ secrets.SC4ES_PRO_TEST_JWT }}" \ + --from-literal=license-public-key="${{ secrets.SC4ES_TEST_PUBLIC_KEY }}" || true + - name: helm install + if: ${{ steps.guard.outputs.skip != 'true' && steps.lic.outputs.skip != 'true' }} + run: | + # The example values point sidecars[].elasticsearch.url at external https://*.example.com + # placeholders. deploy-es.sh stands up in-cluster ES Services; override each URL to the + # matching plaintext in-cluster Service. license/public-key are pointed at sc4es-license + # for the multi-sidecar tier. + extra="" + if [ "${{ matrix.license }}" = "true" ]; then + extra="--set license.secretName=sc4es-license --set license.publicKeySecretName=sc4es-license" + fi + # The three-region/heterogeneous examples set sidecars[].auth.method=bearer pointing at + # per-region sc4es-arrow-* Secrets that this CI does NOT create (the dedicated + # install-secrets job covers Secret-backed auth). Override the federation↔sidecar hop to + # auth.method=none so catalog discovery is what's under test here, not bearer auth — else + # the sidecars enforce bearer with an absent (optional) token and discovery fails. + urls="" + case "${{ matrix.example }}" in + single-cluster) + urls="--set sidecars[0].elasticsearch.url=http://es:9200" ;; + three-region|heterogeneous-ready) + urls="--set sidecars[0].elasticsearch.url=http://es-us-east-1:9200 \ + --set sidecars[1].elasticsearch.url=http://es-eu-west-1:9200 \ + --set sidecars[2].elasticsearch.url=http://es-ap-south-1:9200 \ + --set sidecars[0].auth.method=none \ + --set sidecars[1].auth.method=none \ + --set sidecars[2].auth.method=none" ;; + esac + helm install fed "${CHART_DIR}" \ + -f "${CHART_DIR}/examples/${{ matrix.example }}/values.yaml" \ + ${TEST_ADBC_IMAGE:+--set test.image=$TEST_ADBC_IMAGE} \ + $urls $extra + - name: Wait all pods Ready (sidecars + federation — FACT D) + if: ${{ steps.guard.outputs.skip != 'true' && steps.lic.outputs.skip != 'true' }} + run: kubectl wait --for=condition=Ready pod -l app.kubernetes.io/part-of=softclient4es --timeout=300s + - name: helm test (GetCatalogs == expected — FACT A) + if: ${{ steps.guard.outputs.skip != 'true' && steps.lic.outputs.skip != 'true' }} + run: helm test fed --timeout 180s + - name: Diagnostics on failure + if: ${{ failure() }} + run: | + kubectl get pods -o wide || true + kubectl logs -l app.kubernetes.io/component=federation --tail=200 || true + kubectl logs -l app.kubernetes.io/component=sidecar --tail=100 || true + kubectl logs -l app.kubernetes.io/name=softclient4es-federation -c show-catalogs --tail=100 || true + + # ── 4. PER-ES-VERSION SIDECAR MATRIX — 1 sidecar each (license-free) ──────── + install-es-version: + runs-on: ubuntu-latest + needs: [static-checks, image-availability] + if: ${{ needs.image-availability.outputs.images_published == 'true' }} + strategy: + fail-fast: false + matrix: + es: [6, 7, 8, 9] + steps: + - uses: actions/checkout@v4 + - name: Set vm.max_map_count (Elasticsearch requirement) + run: sudo sysctl -w vm.max_map_count=262144 + - name: Set up Helm + uses: azure/setup-helm@v4 + with: { version: "${{ env.HELM_VERSION }}" } + - name: Create kind cluster + uses: helm/kind-action@v1.10.0 + with: { version: "${{ env.KIND_VERSION }}", node_image: "${{ env.KIND_NODE_IMAGE }}", cluster_name: "fed-es${{ matrix.es }}" } + - name: Pull public images + kind load (federation + ES${{ matrix.es }} sidecar) + run: | + docker pull "docker.io/${FED_IMAGE}:${IMAGE_TAG}" + kind load docker-image "docker.io/${FED_IMAGE}:${IMAGE_TAG}" --name "fed-es${{ matrix.es }}" + img="docker.io/${SIDECAR_IMAGE_PREFIX}${{ matrix.es }}-arrow-flight-sql:${IMAGE_TAG}" + docker pull "$img" + kind load docker-image "$img" --name "fed-es${{ matrix.es }}" + - name: Deploy ES ${{ matrix.es }} + run: ./.github/scripts/deploy-es.sh es-version '${{ matrix.es }}' + - name: helm install (1 sidecar, ES ${{ matrix.es }}, license-free) + run: | + helm install fed "${CHART_DIR}" \ + ${TEST_ADBC_IMAGE:+--set test.image=$TEST_ADBC_IMAGE} \ + --set 'sidecars[0].name=primary' \ + --set "sidecars[0].elasticsearchVersion=${{ matrix.es }}" \ + --set 'sidecars[0].elasticsearch.url=http://es:9200' \ + --set 'sidecars[0].default=true' + - name: Wait Ready + helm test (== 1) + run: | + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/part-of=softclient4es --timeout=300s + helm test fed --timeout 180s + - name: Diagnostics on failure + if: ${{ failure() }} + run: | + kubectl get pods -o wide || true + kubectl logs -l app.kubernetes.io/component=federation --tail=200 || true + kubectl logs -l app.kubernetes.io/component=sidecar --tail=100 || true + + # ── 5. SECRET MANAGEMENT — raw Secret + SealedSecrets (1 sidecar, license-free) ─ + install-secrets: + runs-on: ubuntu-latest + needs: [static-checks, image-availability] + if: ${{ needs.image-availability.outputs.images_published == 'true' }} + strategy: + fail-fast: false + matrix: + backend: [raw-secret, sealed-secrets] + steps: + - uses: actions/checkout@v4 + - name: Set vm.max_map_count (Elasticsearch requirement) + run: sudo sysctl -w vm.max_map_count=262144 + - name: Set up Helm + uses: azure/setup-helm@v4 + with: { version: "${{ env.HELM_VERSION }}" } + - name: Create kind cluster + uses: helm/kind-action@v1.10.0 + with: { version: "${{ env.KIND_VERSION }}", node_image: "${{ env.KIND_NODE_IMAGE }}", cluster_name: "fed-${{ matrix.backend }}" } + - name: Pull public images + kind load (federation + ES8 sidecar) + run: | + docker pull "docker.io/${FED_IMAGE}:${IMAGE_TAG}" + kind load docker-image "docker.io/${FED_IMAGE}:${IMAGE_TAG}" --name "fed-${{ matrix.backend }}" + img="docker.io/${SIDECAR_IMAGE_PREFIX}8-arrow-flight-sql:${IMAGE_TAG}" + docker pull "$img" + kind load docker-image "$img" --name "fed-${{ matrix.backend }}" + - name: Deploy ES 8 + run: ./.github/scripts/deploy-es.sh secrets '8' + - name: Install SealedSecrets controller + materialize the ES Secret (sealed-secrets only) + if: ${{ matrix.backend == 'sealed-secrets' }} + run: ./.github/scripts/install-sealed-secrets.sh es-creds + - name: Create the ES auth Secret (raw-secret only) + if: ${{ matrix.backend == 'raw-secret' }} + run: | + kubectl create secret generic es-creds \ + --from-literal=es-auth-method=basic \ + --from-literal=es-username=elastic --from-literal=es-password=changeme + - name: helm install (1 sidecar, Secret-backed ES auth) + run: | + helm install fed "${CHART_DIR}" \ + ${TEST_ADBC_IMAGE:+--set test.image=$TEST_ADBC_IMAGE} \ + --set 'sidecars[0].name=primary' --set 'sidecars[0].elasticsearchVersion=8' \ + --set 'sidecars[0].elasticsearch.url=http://es:9200' \ + --set 'sidecars[0].elasticsearch.credentialsSecretName=es-creds' \ + --set 'sidecars[0].default=true' + - name: Wait Ready + assert ES env injected + helm test + run: | + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/part-of=softclient4es --timeout=300s + # Assert the Secret-backed env actually reached the sidecar container (16.3 AC9). + kubectl get pod -l app.kubernetes.io/component=sidecar \ + -o jsonpath='{.items[0].spec.containers[0].env[*].name}' | grep -q ELASTIC_CREDENTIALS_USERNAME + helm test fed --timeout 180s + - name: Diagnostics on failure + if: ${{ failure() }} + run: | + kubectl get pods -o wide || true + kubectl logs -l app.kubernetes.io/component=federation --tail=200 || true + kubectl logs -l app.kubernetes.io/component=sidecar --tail=100 || true + + # ── 6. UPGRADE — single-cluster -> three-region (Pro license, best-effort) ── + upgrade: + runs-on: ubuntu-latest + needs: [static-checks, image-availability] + if: ${{ needs.image-availability.outputs.images_published == 'true' }} + steps: + - uses: actions/checkout@v4 + - name: Set vm.max_map_count (Elasticsearch requirement) + run: sudo sysctl -w vm.max_map_count=262144 + - name: Skip if examples missing or no Pro test license (FACT C + FACT F) + id: guard + # Upgrade ends at 3 clusters → same Pro gate as install-topology: JWT secret AND + # public-key secret (+ the published image must be Pro-capable, 16.1 OQ-5). + run: | + if [ ! -f "${CHART_DIR}/examples/single-cluster/values.yaml" ] || [ ! -f "${CHART_DIR}/examples/three-region/values.yaml" ] \ + || [ -z "${{ secrets.SC4ES_PRO_TEST_JWT }}" ] || [ -z "${{ secrets.SC4ES_TEST_PUBLIC_KEY }}" ]; then + echo "::warning::upgrade test needs both examples (16.4) + SC4ES_PRO_TEST_JWT + SC4ES_TEST_PUBLIC_KEY (and a Pro-capable image, FACT F); skipping" + echo "skip=true" >> "$GITHUB_OUTPUT" + fi + - name: Set up Helm + if: ${{ steps.guard.outputs.skip != 'true' }} + uses: azure/setup-helm@v4 + with: { version: "${{ env.HELM_VERSION }}" } + - name: Create kind cluster + if: ${{ steps.guard.outputs.skip != 'true' }} + uses: helm/kind-action@v1.10.0 + with: { version: "${{ env.KIND_VERSION }}", node_image: "${{ env.KIND_NODE_IMAGE }}", cluster_name: fed-upgrade } + - name: Pull public images + kind load + ES + license + install single-cluster + if: ${{ steps.guard.outputs.skip != 'true' }} + run: | + docker pull "docker.io/${FED_IMAGE}:${IMAGE_TAG}" + kind load docker-image "docker.io/${FED_IMAGE}:${IMAGE_TAG}" --name fed-upgrade + for v in 8 9; do + img="docker.io/${SIDECAR_IMAGE_PREFIX}${v}-arrow-flight-sql:${IMAGE_TAG}" + docker pull "$img" + kind load docker-image "$img" --name fed-upgrade + done + ./.github/scripts/deploy-es.sh three-region '8,8,9' + # single-cluster needs a `es` Service too (its sidecar URL is overridden to http://es:9200). + ./.github/scripts/deploy-es.sh secrets '8' + for n in sc4es-license sc4es-pro-license; do + kubectl create secret generic "$n" \ + --from-literal=license-key="${{ secrets.SC4ES_PRO_TEST_JWT }}" \ + --from-literal=license-public-key="${{ secrets.SC4ES_TEST_PUBLIC_KEY }}" || true + done + helm install fed "${CHART_DIR}" -f "${CHART_DIR}/examples/single-cluster/values.yaml" \ + ${TEST_ADBC_IMAGE:+--set test.image=$TEST_ADBC_IMAGE} \ + --set sidecars[0].elasticsearch.url=http://es:9200 + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/part-of=softclient4es --timeout=300s + - name: helm upgrade -> three-region + assert topology change + if: ${{ steps.guard.outputs.skip != 'true' }} + run: | + before=$(kubectl get deploy -l app.kubernetes.io/component=sidecar --no-headers | wc -l) + # Same auth.method=none override as install-topology: this CI tests the topology change + # + catalog count, not the per-region bearer Secrets (which it does not create). + helm upgrade fed "${CHART_DIR}" -f "${CHART_DIR}/examples/three-region/values.yaml" \ + ${TEST_ADBC_IMAGE:+--set test.image=$TEST_ADBC_IMAGE} \ + --set license.secretName=sc4es-license --set license.publicKeySecretName=sc4es-license \ + --set sidecars[0].elasticsearch.url=http://es-us-east-1:9200 \ + --set sidecars[1].elasticsearch.url=http://es-eu-west-1:9200 \ + --set sidecars[2].elasticsearch.url=http://es-ap-south-1:9200 \ + --set sidecars[0].auth.method=none \ + --set sidecars[1].auth.method=none \ + --set sidecars[2].auth.method=none + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/part-of=softclient4es --timeout=300s + after=$(kubectl get deploy -l app.kubernetes.io/component=sidecar --no-headers | wc -l) + [ "$after" -gt "$before" ] || { echo "::error::upgrade did not add sidecar Deployments ($before -> $after)"; exit 1; } + helm test fed --timeout 180s + - name: Diagnostics on failure + if: ${{ failure() && steps.guard.outputs.skip != 'true' }} + run: | + kubectl get pods,deploy -o wide || true + kubectl logs -l app.kubernetes.io/component=federation --tail=200 || true + + # ── 7. UNINSTALL — clean teardown, no leftovers (license-free, 0 sidecars) ── + uninstall: + runs-on: ubuntu-latest + needs: [static-checks, image-availability] + if: ${{ needs.image-availability.outputs.images_published == 'true' }} + steps: + - uses: actions/checkout@v4 + - name: Set up Helm + uses: azure/setup-helm@v4 + with: { version: "${{ env.HELM_VERSION }}" } + - name: Create kind cluster + uses: helm/kind-action@v1.10.0 + with: { version: "${{ env.KIND_VERSION }}", node_image: "${{ env.KIND_NODE_IMAGE }}", cluster_name: fed-uninstall } + - name: Pull public federation image + kind load + run: | + docker pull "docker.io/${FED_IMAGE}:${IMAGE_TAG}" + kind load docker-image "docker.io/${FED_IMAGE}:${IMAGE_TAG}" --name fed-uninstall + - name: Install (0-sidecar — license-free, no ES needed) then uninstall + run: | + helm install fed "${CHART_DIR}" + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=softclient4es-federation --timeout=180s + helm uninstall fed + sleep 5 + leftover=$(kubectl get all,configmap -l app.kubernetes.io/instance=fed --no-headers 2>/dev/null | grep -v '^kubernetes ' | wc -l) + [ "$leftover" = "0" ] || { echo "::error::helm uninstall left $leftover resources"; kubectl get all,configmap -l app.kubernetes.io/instance=fed; exit 1; } diff --git a/README.md b/README.md index 763e47a..d00ba5e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,11 @@ # softclient4es-helm -The SoftClient4ES Helm charts + +Helm charts for the [SoftClient4ES](https://softclient4es.dev/) ecosystem. + +## Charts + +- [`softclient4es-federation/`](softclient4es-federation/) — deploys the SoftClient4ES + multi-cluster federation Arrow Flight SQL server plus optional per-Elasticsearch-version + sidecars. + +See each chart's own `README.md` for installation and configuration details. diff --git a/softclient4es-federation/.helmignore b/softclient4es-federation/.helmignore new file mode 100644 index 0000000..4a65851 --- /dev/null +++ b/softclient4es-federation/.helmignore @@ -0,0 +1,12 @@ +# Patterns to ignore when building Helm packages. +# tests/golden/ is intentionally NOT ignored — the golden baseline is committed. +.DS_Store +.git/ +.gitignore +*.swp +*.bak +*.tmp +*.orig +*.tmproj +.idea/ +.vscode/ diff --git a/softclient4es-federation/Chart.yaml b/softclient4es-federation/Chart.yaml new file mode 100644 index 0000000..340fa3f --- /dev/null +++ b/softclient4es-federation/Chart.yaml @@ -0,0 +1,33 @@ +apiVersion: v2 +name: softclient4es-federation +description: >- + SoftClient4ES multi-cluster federation — an Arrow Flight SQL server that + federates SQL queries (incl. cross-cluster JOINs) across one or more + per-Elasticsearch-version sidecars. This chart deploys the ES-agnostic + federation server; sidecars are added in a later chart revision. +type: application +# Chart version — bump on every chart change (SemVer). +# 0.2.0: per-ES-version sidecars + federation servers ConfigMap (Story 16.2). +# 0.3.0: Kubernetes Secrets (key-name contract + per-key override + envFrom), +# Secret-backed federation→sidecar creds (CONFIG_FORCE_*), TLS/Ingress (Story 16.3). +version: 0.3.0 +# The federation application/image version this chart deploys by default. +# Keep in sync with docker.io/softnetwork/softclient4es-federation:. +# NOTE: this appVersion is the future PUBLISHED-release tag (OQ-2) of the public +# DockerHub federation image — set it to whatever tag the release/CI push (OQ-1) +# produces; until then it is PROPOSED. +appVersion: "0.2.0" +keywords: + - elasticsearch + - flight-sql + - federation + - arrow + - sql + - duckdb +home: https://softclient4es.dev +sources: + - https://github.com/SOFTNETWORK-APP/softclient4es-helm +maintainers: + - name: SOFTNETWORK + url: https://softclient4es.dev +icon: https://softclient4es.dev/images/logo.png diff --git a/softclient4es-federation/README.md b/softclient4es-federation/README.md new file mode 100644 index 0000000..78a8427 --- /dev/null +++ b/softclient4es-federation/README.md @@ -0,0 +1,416 @@ +# SoftClient4ES Federation Helm Chart + +Deploys the ES-agnostic SoftClient4ES federation Arrow Flight SQL server plus, +optionally, one per-Elasticsearch-version Arrow Flight SQL **sidecar** per backing +Elasticsearch cluster (see the "Sidecars" section). + +With the default `sidecars: []` the chart stands up a **single federation Pod** with +**no downstream servers** (`arrow.flight.federation.servers = {}`). Add entries to +`sidecars[]` to federate one or more (mixed-version) Elasticsearch clusters. Chart +0.3.x adds Kubernetes-Secret-backed credentials (incl. Secret-backed federation→sidecar +auth), TLS/Ingress termination, and secret-backend examples (see "Secrets, TLS & Ingress"). +Later revisions add topology examples, CI smoke tests, and the full operator guide. + +## Prerequisites + +- Kubernetes 1.24+ (1.27+ recommended; native gRPC readiness probes land in chart 0.2.x). +- Helm 3.x. +- A container runtime able to pull from public DockerHub (`docker.io`). +- (Optional) a Kubernetes Secret holding the license JWT / API key — see `license.secretName`. + +## Install + +```sh +helm install fed ./softclient4es-federation +# or with overrides +helm install fed ./softclient4es-federation -f my-values.yaml +``` + +## Upgrade + +```sh +helm upgrade fed ./softclient4es-federation -f my-values.yaml +``` + +## Rollback + +```sh +helm history fed +helm rollback fed +``` + +## Uninstall + +```sh +helm uninstall fed +``` + +## Notes + +- **Health is gRPC, not HTTP.** This release runs the federation with NO downstream + servers (`servers={}`). The health endpoint is the gRPC `grpc.health.v1.Health` + service on port `32021` and reports `NOT_SERVING` until at least one sidecar is + configured (chart 0.2.x). Therefore liveness/readiness here are **TCP-socket + probes** (liveness → health port `32021`, readiness → Flight SQL port `32020`) + that pass once the process is listening. Chart 0.2.x switches readiness to a + native gRPC probe once sidecars make the `SERVING` aggregate meaningful (GA on + Kubernetes 1.27+). +- **Deployable unlicensed (Community).** With `license.secretName` empty the + federation boots in Community mode and reaches Ready (a no-downstream federation + is within the Community single-cluster quota). Add `license.secretName` once you + configure sidecars (0.2.x), where the cluster quota begins to apply. +- **Image availability.** The federation image + `docker.io/softnetwork/softclient4es-federation:` is published to public + DockerHub at R1 release (release/CI concern, OQ-1). Pin `image.tag` to the + published release tag in your `values.yaml` (`--set image.tag=`); the chart + default falls back to the Chart.yaml `appVersion`. +- **Read-only root filesystem.** `securityContext.readOnlyRootFilesystem: true` is + paired with a writable `emptyDir` mounted at `/tmp`. This is **required for boot**: + the DuckDB JDBC driver extracts its native library to `java.io.tmpdir` on startup + even with `federation.duckdb.path: ":memory:"`. A file `duckdb.path` MUST point at + a writable mount (under `/tmp` or a PersistentVolume). +- **Configuration reference.** Every value maps to an `arrow.flight.federation.*` / + `FEDERATION_*` env var (see `values.yaml` comments). + +## Configuration + +| Key | Default | Description | +| --- | --- | --- | +| `replicaCount` | `1` | Number of federation Pods. | +| `image.repository` | `docker.io/softnetwork/softclient4es-federation` | Federation image repository. | +| `image.tag` | `""` (→ `.Chart.AppVersion`) | Image tag; empty falls back to `appVersion`. | +| `image.pullPolicy` | `IfNotPresent` | Image pull policy. | +| `federation.maxMemory` | `512m` | `FEDERATION_MAX_MEMORY` — DuckDB `memory_limit`. | +| `federation.queryTimeoutSeconds` | `30` | `FEDERATION_QUERY_TIMEOUT`. | +| `federation.health.port` | `32021` | `FEDERATION_HEALTH_PORT` (gRPC health). | +| `federation.health.probeTimeoutSeconds` | `5` | `FEDERATION_HEALTH_PROBE_TIMEOUT`; also K8s probe `timeoutSeconds`. | +| `federation.duckdb.path` | `:memory:` | `FEDERATION_DUCKDB_PATH`. | +| `federation.upgradeUrl` | `https://portal.softclient4es.com/pricing` | `FEDERATION_UPGRADE_URL`. | +| `telemetry.enabled` | `true` | `SOFTCLIENT4ES_TELEMETRY_ENABLED` daily-ping opt-out (`false` opts out). | +| `license.secretName` | `""` | Secret holding license/API key; empty = Community. | +| `license.publicKeySecretName` | `""` | Secret holding the Ed25519 public JWK for OFFLINE license verification → `SOFTCLIENT4ES_LICENSE_PUBLIC_KEY`; empty = use JWKS fetch. | +| `license.publicKeyKey` | `license-public-key` | Data key within `license.publicKeySecretName`. | +| `service.type` | `ClusterIP` | Service type. | +| `service.port` | `32020` | `FEDERATION_PORT` (Flight SQL); the only port exposed by the Service. | +| `resources` | req `1Gi`/`500m`, lim `2Gi`/`1000m` | Container resource requests/limits. | +| `scratch.sizeLimit` | `2Gi` | Size limit of the writable `/tmp` `emptyDir`. | +| `federation.probes.useGrpc` | `true` | When sidecars exist, use a native gRPC readiness probe (all-or-nothing — see below). `false` keeps TCP readiness. | +| `federation.tls.enabled` | `false` | Add a `tls:` block to the Ingress (TLS terminates at the Ingress; the pod is plaintext). See "Secrets, TLS & Ingress". | +| `federation.tls.secretName` | `""` | A `kubernetes.io/tls` Secret (`tls.crt`+`tls.key`), e.g. cert-manager-issued. | +| `federation.credentialsFromEnv` | `true` | Inject Secret-backed federation→sidecar creds via `CONFIG_FORCE_*` (`override_with_env_vars`). | +| `ingress.enabled` | `false` | Render an Ingress for the federation Flight SQL endpoint (gRPC — needs a gRPC-capable controller). | +| `ingress.className` | `""` | `spec.ingressClassName` (e.g. `nginx`). | +| `ingress.annotations` | `{}` | Free-form annotations (cert-manager / external-DNS / `backend-protocol: "GRPC"`). | +| `ingress.hosts` | (see `values.yaml`) | Host/path rules; an empty `host` renders no rule. | +| `ingress.tls` | `[]` | Explicit Ingress `tls:` entries; empty + `federation.tls.enabled` auto-fills from `federation.tls.secretName`. | +| `sidecars` | `[]` | Per-ES-version sidecars — see the "Sidecars" section below. | +| `sidecarDefaults` | (see `values.yaml`) | Shared resource/probe/security defaults applied to every sidecar. | +| `test.image` | `""` (→ `python:3.12-slim`) | Image for the `helm test` smoke Job; pin a pre-baked ADBC image for air-gapped clusters. | +| `test.adbcVersion` | `1.6.0` | ADBC driver version the smoke Job `pip install`s at runtime (when `test.image` is not pre-baked). | + +## Sidecars — federating one or more Elasticsearch clusters + +Each entry in `sidecars[]` deploys an Arrow Flight SQL gateway in front of ONE ES +cluster and registers it with the federation. Sidecars may target DIFFERENT ES +major versions (6/7/8/9) in the same deployment — the image is chosen automatically +from `elasticsearchVersion`. With `sidecars: []` (the default) the chart behaves +exactly like the 0.1.x federation-only skeleton (no Deployments/Services, no ConfigMap, +federation readiness stays TCP). + +The backing-ES endpoint is given as a single `elasticsearch.url` (`scheme://host:port`), +which the chart decomposes into the `ELASTIC_SCHEME`/`ELASTIC_HOST`/`ELASTIC_PORT` env +the sidecar reads (there is no single ES-URL env var). A scheme-less url defaults to +`http`, a port-less url to `9200`; for a TLS or non-9200 cluster, supply the scheme/port +in the url or set explicit `elasticsearch.scheme`/`.host`/`.port`. + +### Add a cluster +1. Append an entry to `sidecars[]` (name, elasticsearchVersion, elasticsearch.url, + credentials Secret). +2. `helm upgrade fed ./softclient4es-federation -f my-values.yaml` + +The federation ConfigMap is re-rendered and the federation Pod rolls automatically +(a `checksum/config` annotation forces the restart so the new cluster is picked up). + +### Remove a cluster +1. Delete its entry from `sidecars[]`. +2. `helm upgrade …` — its Deployment + Service are removed; the federation ConfigMap + is re-rendered and the federation Pod rolls (checksum/config annotation). + +### Mixed ES versions +Fully supported — e.g. ES 8 in `us`, ES 9 in `eu`. Common during a version +migration: run mixed-version federation while migrating one cluster at a time. + +### ⚠️ Licensing: a Pro/Enterprise license is REQUIRED once you have 2+ sidecars +The federation enforces a per-platform cluster QUOTA at startup. The quota — not a +feature flag — is the gate: + +| Sidecars | Community (no license) | Pro | Enterprise | +|---|---|---|---| +| 1 | ✅ boots Ready (maxClusters=1) | ✅ | ✅ | +| 2–5 | ❌ federation CrashLoops (sys.exit) | ✅ (maxClusters=5) | ✅ | +| 6+ | ❌ | ❌ | ✅ (unlimited) | + +So a **single-cluster** federation runs with NO license. The moment you add a +**second** sidecar you MUST set `license.secretName` to a **Pro** (≤5 clusters) or +**Enterprise** (unlimited) license, or the federation Pod CrashLoops by design. +(The Federation *feature* is present in all tiers; the limit is `maxClusters`.) + +### Per-sidecar / federation auth (single source of truth) +A single `sidecars[].auth` block drives BOTH the sidecar's incoming `ARROW_AUTH_*` +auth AND the federation's outgoing `servers..credentials`. `method: none` +(default, intra-cluster trust) is the common case. For `basic`/`bearer`/`apikey`, the +SAME `auth.credentialsSecretName` Secret now feeds both sides (chart 0.3.x): the sidecar +reads `ARROW_AUTH_*` and the federation receives the value via `CONFIG_FORCE_*` +(`override_with_env_vars`) — see "Secrets, TLS & Ingress" below. Inline values are still +accepted for dev/test; the template fails fast with an actionable message if you set a +non-`none` method with neither a Secret nor inline creds. + +### Cross-namespace deployments +Default is same-namespace (the ConfigMap uses `..svc...`). +Cross-namespace federation is possible by deploying sidecars in another namespace and +overriding the host; not the default — see the operator guide. + +### ⚠️ Readiness behavior with multiple sidecars (`federation.probes.useGrpc`) +By default (`useGrpc: true`, K8s ≥ 1.27) the federation's gRPC readiness aggregate is +**all-or-nothing**: if **ANY ONE** downstream sidecar is unreachable, the federation Pod +goes **NotReady** and is removed from its Service — so **every** federation query fails, +including ones targeting the still-healthy sidecars. This is "fail-closed" routing. For +multi-sidecar production where partial availability is preferable, set +`federation.probes.useGrpc: false` to keep the (process-listening) TCP readiness — the +federation stays Ready and degrades per-query instead of dropping entirely. On K8s < 1.27 +you MUST use `useGrpc: false` (native gRPC probes are GA only from 1.27; `grpc_health_probe` +exec is the compat path — see the operator guide). + +### `helm test` smoke +With sidecars configured, `helm test fed` runs a Job that connects to the federation +Flight SQL endpoint and asserts `GetCatalogs` returns one catalog per sidecar +(`len(sidecars)`). The 2-sidecar smoke requires a Pro/Enterprise license (the quota +gate above) and reachable backing ES for each sidecar. The Job **retries the connect + +`GetCatalogs` for up to ~60 s** — the federation is NotReady until it has discovered every +downstream (gRPC readiness is all-or-nothing), so a fresh install needs a few seconds. + +The test Pod defaults to `python:3.12-slim` and `pip install`s the ADBC driver at runtime +(needs PyPI reachability). For air-gapped or rate-limited clusters, pin a pre-baked ADBC image +with `--set test.image=` (and `--set test.adbcVersion=` to control the driver version +when the runtime install IS used). This is the same Job CI runs. + +> **Offline license verification (`license.publicKeySecretName`).** When the federation must +> verify a license JWT WITHOUT reaching the license server's JWKS endpoint (air-gapped clusters, +> or a JWT whose `kid` is not in the prod JWKS), set `license.publicKeySecretName` to a Secret +> whose `license.publicKeyKey` data key holds the matching Ed25519 public JWK. It is mounted as +> `SOFTCLIENT4ES_LICENSE_PUBLIC_KEY` (the air-gap path in the license verifier). Leave empty (the +> default) to use the normal JWKS fetch — it renders nothing, so the golden render is unaffected. + +## Secrets, TLS & Ingress + +The chart **references** Kubernetes Secrets by name and never creates them. See +[`docs/secret-backends.md`](docs/secret-backends.md) for how to create them (raw / +SealedSecrets / ESO / Vault), and `examples/sealed-secrets/` + `examples/external-secrets/` +for ready-to-adapt manifests. + +### Secret key-name contract +Each referenced Secret must carry these data keys (override per-sidecar via `secretKeys`): + +| values.yaml field | Secret data keys | +|---|---| +| `sidecars[].elasticsearch.credentialsSecretName` | `es-auth-method`, `es-username`, `es-password`, `es-api-key`, `es-bearer-token` | +| `sidecars[].auth.credentialsSecretName` | `arrow-username`, `arrow-password`, `arrow-bearer-token`, `arrow-api-key` | +| `license.secretName` | `license-key`, `api-key` | +| `federation.tls.secretName` | `tls.crt`, `tls.key` (`kubernetes.io/tls`) | + +Override the data-key names per sidecar with `elasticsearch.secretKeys` / `auth.secretKeys`, +or mount the whole Secret as env with `useEnvFrom: true` (the Secret's keys must then BE the +env-var names — no remapping). + +> **A Secret with the WRONG keys is silent at install time.** The chart's `secretKeyRef`s are +> `optional: true`, so a key-name mismatch (or a Secret not created/synced yet) does NOT fail +> `helm install` — the env is simply absent, and the federation then CrashLoops at boot with a +> `FlightCredentials`/`validate()` credentials error (the sidecar may start but fail its ES/auth +> connection). If a pod CrashLoops right after a Secret-backed install, check: the Secret EXISTS +> (`kubectl get secret `), its data keys MATCH this table +> (`kubectl get secret -o jsonpath='{.data}'`), and — for ESO/SealedSecrets — it has +> MATERIALIZED (`kubectl get externalsecret` / `kubectl get sealedsecret`, sealed for THIS +> namespace). An ESO sync-lag CrashLoop **self-heals** on the next restart once the Secret +> appears — do not uninstall prematurely. + +### Federation → sidecar credentials (single Secret, both sides) +ONE `sidecars[].auth.credentialsSecretName` feeds BOTH the sidecar's incoming `ARROW_AUTH_*` +AND the federation's outgoing auth to that sidecar. Because a ConfigMap cannot read a Secret, +the federation receives the credential via Typesafe Config `override_with_env_vars`: the chart +sets `-Dconfig.override_with_env_vars=true` and injects +`CONFIG_FORCE_arrow_flight_federation_servers__credentials_` env from the Secret +(toggle: `federation.credentialsFromEnv`, default `true`). The ConfigMap renders only the +`method` (not secret); the value arrives from the Secret. **Sidecar names must be strict +RFC1123 labels** — the name is mangled into the `CONFIG_FORCE_*` path, and a `_`/`.`/uppercase +would silently mis-target it (the template fails fast on a non-RFC1123 name). `auth.useEnvFrom` +(whole-Secret mode) is **incompatible** with a Secret-backed federation→sidecar credential — +the federation reads the Secret per-key (`arrow-bearer-token`, …), which a whole-Secret-shaped +Secret does not carry — so the chart fails fast on that combination; use the per-key default +(`useEnvFrom: false`) for any non-`none` Secret-backed auth method. + +> **Rotation needs a restart.** Env-from-Secret is read at Pod start, so rotating the Secret +> value needs `kubectl rollout restart deploy/` (or a reloader of your choice). When +> ONE Secret feeds BOTH the sidecar (`ARROW_AUTH_*`) and the federation (`CONFIG_FORCE_*`), +> restart BOTH Deployments or the two sides drift (the federation presents the old credential +> to a sidecar that now expects the new one). + +### `auth.method=none` + a Secret (benign) +Setting `auth.credentialsSecretName` while `auth.method: none` is harmless — the sidecar's +server-side auth is off, the injected `ARROW_AUTH_*` env are ignored, and the federation emits +no `CONFIG_FORCE_*` for that sidecar. The chart does NOT fail this (you may pre-stage a Secret +before flipping `method`); just note the Secret has no effect until `method` is non-`none`. + +### TLS (at the Ingress, not the Pod) +The federation Flight SQL server listens **plaintext gRPC only** — it does NOT terminate TLS. +Terminate TLS at an Ingress/gateway: set `federation.tls.enabled` + `federation.tls.secretName` +(a cert-manager `kubernetes.io/tls` Secret) and `ingress.enabled`. Flight SQL is gRPC, so the +Ingress controller MUST proxy gRPC backends with `backend-protocol: "GRPC"` (**NOT** `GRPCS` — +the pod is plaintext; the Ingress terminates TLS and forwards cleartext h2c), or use a +Gateway-API gateway. nginx-ingress speaks HTTP/2 to clients only over a TLS listener, so +Flight-SQL-over-Ingress in practice means TLS-at-the-edge; for plaintext in-cluster access, hit +the ClusterIP/LoadBalancer Service (`32020`) directly rather than a plaintext Ingress. +`sidecars[].tls: true` is a SEPARATE knob — it makes the federation connect to THAT sidecar over +TLS (the outgoing hop), unrelated to the federation's own inbound edge TLS. + +### Example currency +The `examples/external-secrets/` use `external-secrets.io/v1` (the stable API; `v1beta1` was +removed at ESO v0.17.0). The `examples/sealed-secrets/` reference the post-move repo +`bitnami.github.io` and are NON-FUNCTIONAL placeholders that MUST be re-sealed per cluster (and +per namespace — SealedSecrets are namespace-scoped by default). + +## Regenerating the golden render + +The committed render baselines under `tests/golden/` let a future PR detect template +drift: `default.yaml` (0 sidecars), `two-sidecars.yaml` (federation + 2 mixed ES8/ES9 +sidecars), `secret-auth.yaml` (1 sidecar, Secret-backed ES + sidecar bearer auth + +federation `CONFIG_FORCE_*`), and `ingress-tls.yaml` (federation behind a cert-manager +TLS Ingress). The `example-*.yaml` baselines render the three topology examples under +`examples/` (Story 16.4). Regenerate (and review the diff) with: + +```sh +helm template fed ./softclient4es-federation > ./softclient4es-federation/tests/golden/default.yaml +helm template fed ./softclient4es-federation -f ./softclient4es-federation/tests/values/two-sidecars.yaml \ + > ./softclient4es-federation/tests/golden/two-sidecars.yaml +helm template fed ./softclient4es-federation -f ./softclient4es-federation/tests/values/secret-auth.yaml \ + > ./softclient4es-federation/tests/golden/secret-auth.yaml +helm template fed ./softclient4es-federation -f ./softclient4es-federation/tests/values/ingress-tls.yaml \ + > ./softclient4es-federation/tests/golden/ingress-tls.yaml +# Topology examples (Story 16.4): +helm template fed ./softclient4es-federation -f ./softclient4es-federation/examples/single-cluster/values.yaml \ + > ./softclient4es-federation/tests/golden/example-single-cluster.yaml +helm template fed ./softclient4es-federation -f ./softclient4es-federation/examples/three-region/values.yaml \ + > ./softclient4es-federation/tests/golden/example-three-region.yaml +helm template fed ./softclient4es-federation -f ./softclient4es-federation/examples/heterogeneous-ready/values.yaml \ + > ./softclient4es-federation/tests/golden/example-heterogeneous-ready.yaml +git diff --stat ./softclient4es-federation/tests/golden/ +``` + +Any diff must be intentional. CI (Story 16.5) enforces these goldens plus `helm lint` +and `kubeconform -strict` (including the 2-sidecar, Secret-backed, and TLS/Ingress renders). +The chart `templates/` must emit ZERO `kind: Secret` — assert with +`helm template fed ./softclient4es-federation -f … | grep -c '^kind: Secret'` (expected `0`). + +> **`example-three-region.yaml` and `example-heterogeneous-ready.yaml` are BYTE-IDENTICAL +> by design** — `diff` between them is EMPTY (Story 16.4 Decision A2). The two `values.yaml` +> overlays carry the SAME active values (same license Secret, telemetry, `useGrpc`, and the +> same three `sidecars[]`); they differ ONLY in comments and the commented-out R2b +> `duckdb-attach` preview, and Helm strips all comments before rendering. The ONLY signal +> distinguishing the two examples is the SOURCE: `examples/heterogeneous-ready/values.yaml` +> contains the `type = "duckdb-attach"` R2b marker and `examples/three-region/values.yaml` +> does not (Decision A2b) — CI (16.5) greps for this. A regression that overwrote one +> overlay with the other would pass the golden + `helm test` gates but FAIL the grep. + +## Topology examples + +Copy-pasteable `values.yaml` overlays for three operator scenarios live under `examples/` +(Story 16.4). Each has its own `README.md` with a "when to use", an ASCII topology diagram, +the install command, and the `SHOW CATALOGS` smoke expectation: + +| Example | Topology | License | `SHOW CATALOGS` | +|---|---|---|---| +| `examples/single-cluster/` | federation + 1 ES8 sidecar | **none** (Community, `maxClusters=1`) | 1 | +| `examples/three-region/` | federation + us-east-1 (ES8) / eu-west-1 (ES8) / ap-south-1 (ES9) | **Pro / Enterprise** | 3 | +| `examples/heterogeneous-ready/` | three-region today + a commented R2b `duckdb-attach` preview (PG/MySQL/Snowflake) | **Pro / Enterprise** | 3 (R2b inactive) | + +Install any of them with `helm install softclient4es-federation softclient4es-federation -f examples//values.yaml` +(create the referenced Secrets first — see each example's README). + +## CI / CD coverage (Story 16.5) + +Every PR touching `softclient4es-federation/**` runs +[`.github/workflows/federation-helm.yml`](../.github/workflows/federation-helm.yml): static +checks (lint + template + `kubeconform -strict` + golden-file diff), an image-availability +prerequisite (pulls the public DockerHub federation + sidecar images), and per-topology / +per-ES-version / secret-backend / upgrade / uninstall installs on ephemeral `kind` clusters. +The smoke test is the chart's own `helm test` Job (ADBC Flight SQL `GetCatalogs`, asserting one +catalog per sidecar) — customers run the exact same `helm test fed` post-install that CI runs. + +This chart repo is **chart-only** (no sbt build): CI never builds images. The live-install jobs +`docker pull` the public DockerHub images and `kind load` them. Until those images are published +(OQ-1), the live-install jobs **probe image availability and skip with a CI annotation** rather +than fail — the static-validation gates run unconditionally on every PR with zero external deps. + +### Tested vs best-effort matrix + +| Scenario | Gate | Tier | License | Notes | +|---|---|---|---|---| +| `helm lint` (chart + each example) | static | **tested** (blocker) | none | unconditional | +| `helm template` + `kubeconform -strict` | static | **tested** (blocker) | none | per example | +| Golden-file diff (template drift) | static | **tested** (blocker) | none | per example + the 16.1/16.2/16.3 goldens | +| Zero `kind: Secret` rendered | static | **tested** (blocker) | none | 16.3 contract | +| `heterogeneous-ready` discriminator | static | **tested** (blocker) | none | greps the `type = "duckdb-attach"` marker (the only signal vs `three-region` — the goldens are byte-identical) | +| Install single-cluster (1×ES8) | kind | **tested** | none (Community) | `GetCatalogs` == 1 | +| Install three-region (2×ES8 + 1×ES9) | kind | **best-effort\*** | **Pro** | mixed-version; `== 3` | +| Install heterogeneous-ready (3×ES) | kind | **best-effort\*** | **Pro** | R2b placeholders inactive; `== 3` | +| Per-ES-version sidecar (6/7/8/9) | kind | **tested** | none | 1 sidecar each; `== 1` | +| Secret backend: raw K8s Secret | kind | **tested** | none | 1 sidecar Secret-backed | +| Secret backend: SealedSecrets | kind | **tested** | none | controller installed + re-sealed in-cluster | +| Secret backend: External Secrets Operator (ESO) | — | **best-effort / documented** | — | needs an external store; examples shipped (16.3), not CI-run | +| Secret backend: Vault Agent Injector | — | **best-effort / documented** | — | needs Vault; documented (16.3) | +| Upgrade single → three-region | kind | **best-effort\*** | **Pro** | topology change asserted (+2 sidecar Deployments) | +| Uninstall clean | kind | **tested** | none | `kubectl get all` empty | + +\* The multi-cluster (Pro) tiers run only when ALL of: (1) the federation image bundles the +JWT-verifying SPI (a **Pro-capable** image — an OSS-only image ships only the Community SPI and +cannot verify ANY Pro JWT), (2) the `SC4ES_PRO_TEST_JWT` repo secret, and (3) the +`SC4ES_TEST_PUBLIC_KEY` repo secret (injected as `SOFTCLIENT4ES_LICENSE_PUBLIC_KEY` via +`license.publicKeySecretName` so the JWT verifies offline) are present. Otherwise they are +**skipped with a CI annotation** (not a failure). A **single-cluster** federation is license-FREE +(Community `maxClusters=1`) and is always tested; the static three-region/heterogeneous golden +proves the mixed-version RENDER on every PR even when the live multi-cluster install is skipped. + +### Image tags in CI + +The live-install jobs resolve the chart's DEFAULT image refs (federation `image.tag:""` → +`appVersion`; sidecar tag → `appVersion`), `docker pull` those public DockerHub tags, and +`kind load` them — no per-image `--set image.tag` override, so the federation + sidecar tags +stay consistent with the committed goldens (which are `appVersion` renders). Until the images +are published (OQ-1), an availability probe (`docker manifest inspect`) gates each live-install +job: if a required image is absent the job is **skipped with a `::warning::` annotation**, never +failed. The golden gate renders with NO `--set image.tag`. + +### CI failure modes (troubleshooting) + +| CI job fails with… | Likely cause | Fix | +|---|---|---| +| golden-file diff non-empty | a template change was not regenerated | run the regen commands above; commit the new golden if the change is intentional | +| `kubeconform` invalid resource | a manifest field renamed/typo (e.g. `replicaCount` vs `replicas`) | fix the template; `replicas` is the Deployment field | +| `heterogeneous-ready` discriminator fails | `three-region` and `heterogeneous-ready` overlays drifted (one copied over the other) | restore the commented R2b `duckdb-attach` preview in `heterogeneous-ready` (the goldens are byte-identical — this grep is the only signal) | +| federation pod CrashLoopBackOff (3 sidecars, license supplied but Community at runtime) | the image lacks the JWT SPI — it cannot verify the Pro JWT → falls back to Community → `maxClusters=1` exceeded | use a Pro-CAPABLE federation image (JWT SPI on classpath, 16.1 OQ-5); injecting a JWT into an OSS-only image does nothing | +| federation pod CrashLoopBackOff (`InvalidLicense: Unknown key ID: …`) | the JWT verification key didn't resolve (no JWKS entry for the kid AND no `SOFTCLIENT4ES_LICENSE_PUBLIC_KEY`) | set `license.publicKeySecretName` → `SOFTCLIENT4ES_LICENSE_PUBLIC_KEY` (the offline verifier path), or ensure the license-server JWKS carries the kid | +| federation pod CrashLoopBackOff (3 sidecars, no license at all) | Community `maxClusters=1` exceeded | supply a Pro/Enterprise license (`license.secretName`) — by design | +| federation pod CrashLoopBackOff (`validate()` / `FlightCredentials`) | Secret-backed cred didn't arrive (wrong key / ESO sync lag) | check the Secret exists + keys match the contract table above; self-heals on next restart | +| federation NotReady, smoke connect-refused | a sidecar's backing ES is down/unreachable (all-or-nothing gRPC readiness) | ensure every sidecar's ES is reachable; or set `federation.probes.useGrpc=false` for partial availability | +| `helm test` count mismatch | a sidecar failed discovery (ES down, or wrong `elasticsearch.url`) | check sidecar + ES logs; verify each sidecar's `elasticsearch.url` resolves in-cluster | +| image pull error (federation OR sidecar) | the public DockerHub tag is not published yet (OQ-1) | the availability probe should skip the live-install job until the image is published; once published, the job `docker pull`s + `kind load`s the public tag | +| live-install job skipped with a `::warning::` | the required public DockerHub image is not yet published (OQ-1) | expected until release; the static-validation gates still run unconditionally | +| ES pod crash (`max virtual memory areas …`) | `vm.max_map_count` too low | `sudo sysctl -w vm.max_map_count=262144` on the runner before ES starts (the workflow does this) | + +### Duration & sharding + +Static checks < 2 min; image pull ~1–2 min; each install job ~6–9 min (when the public images +exist — otherwise the live jobs skip in seconds). With the matrix in parallel, wall-clock ≈ +image-pull + slowest install ≈ ~12 min (< 30 min budget). If runner contention serializes the +matrix past 30 min, move the per-ES-version + secret-backend jobs to a `schedule:` nightly +trigger (uncomment the `schedule:` block in the workflow) and keep PRs to static-checks + +single-cluster + three-region + uninstall. diff --git a/softclient4es-federation/docs/operator-guide.md b/softclient4es-federation/docs/operator-guide.md new file mode 100644 index 0000000..35e9d91 --- /dev/null +++ b/softclient4es-federation/docs/operator-guide.md @@ -0,0 +1,225 @@ +# SoftClient4ES Federation — Operator Guide + +> **Authoritative guide.** This is the canonical operator guide that ships with the federation Helm chart at `softclient4es-federation/docs/operator-guide.md`. The web and core-docs copies point here. + +The SoftClient4ES federation Helm chart deploys a **cross-cluster Arrow Flight SQL coordinator** (the *federation server*) plus one **per-ES-version sidecar** for each Elasticsearch cluster you want to federate. One `values.yaml` + `helm install` replaces 30+ minutes of hand-written HOCON, Kubernetes manifests, Secrets, and probe wiring. + +## 1. Prerequisites + +- A Kubernetes cluster (v1.27+ recommended for native gRPC readiness probes; on **older clusters set `federation.probes.useGrpc: false`** for TCP readiness — no extra binary needed; see §9/§10). +- **Helm 3**. +- One or more Elasticsearch clusters **reachable from the K8s cluster** (each sidecar opens a connection to exactly one ES cluster of its compiled-against major version). +- A container runtime able to pull from **public DockerHub** (`docker.io/softnetwork/...`). For rate-limit avoidance on first install, configure `imagePullSecrets` with an authenticated DockerHub account. +- (Multi-cluster only) a **Pro or Enterprise license** — see §7. Single-cluster federation runs free on the Community tier. + +> **Image availability:** the federation image `docker.io/softnetwork/softclient4es-federation` is published to public DockerHub at R1 release. The four sidecar images `docker.io/softnetwork/softclient4es{6,7,8,9}-arrow-flight-sql` are already published. Always pin `image.tag` to the published release tag in your `values.yaml`. + +**Working directory convention.** Every `cp examples/…` and `helm install … ./softclient4es-federation` command in this guide is run **from the chart directory `softclient4es-federation/`** (where `Chart.yaml`, `examples/`, and the `./softclient4es-federation` chart path resolve). `cd` there first: + +```bash +cd softclient4es-federation/ # all commands below run from here +``` + +The release name you pass to `helm install …` (this guide uses `fed`) becomes part of the in-cluster Service DNS — `-softclient4es-federation..svc.cluster.local`. Keep it consistent across `install`, `upgrade`, `test`, and `rollback`. + +## 2. The 5-minute path — single cluster + +```bash +# 1. Copy the starter example +cp examples/single-cluster/values.yaml my-values.yaml + +# 2. Edit my-values.yaml — set the one sidecar's ES coordinates + version: +# sidecars[0].elasticsearchVersion: 8 +# sidecars[0].elasticsearch.url: https://es.mycorp.internal:9200 +# sidecars[0].elasticsearch.credentialsSecretName: my-es-secret # see §6 +# image.tag: + +# 3. Create your ES credentials Secret (raw example; §6 for SealedSecrets/ESO/Vault) +kubectl create secret generic my-es-secret \ + --from-literal=es-auth-method=basic \ + --from-literal=es-username=elastic \ + --from-literal=es-password='' + +# 4. Install +helm install fed ./softclient4es-federation -f my-values.yaml + +# 5. Validate +helm test fed # runs GetCatalogs; passes when the federation lists 1 catalog +``` + +Single-cluster is **license-free** (Community `maxClusters=1`). The federation exposes Flight SQL on `ClusterIP` port **32020**; reach it in-cluster at `fed-softclient4es-federation..svc.cluster.local:32020`, or expose it via an Ingress (§6, TLS). + +See [`examples/single-cluster/README.md`](../examples/single-cluster/README.md) for the full single-cluster narrative + topology diagram. + +## 3. Multi-cluster — three regions, mixed ES versions + +```bash +cp examples/three-region/values.yaml my-values.yaml +# Per-region sidecars (us-east-1 ES8 default · eu-west-1 ES8 · ap-south-1 ES9), +# each with its own elasticsearch.credentialsSecretName + auth.credentialsSecretName. + +# 1. Create the Pro/Enterprise license Secret (REQUIRED — 3 clusters > Community's quota of 1; see §7). +# Replace with the JWT from the portal. +kubectl create secret generic sc4es-pro-license \ + --from-literal=license-key='' + +# 2. Create the per-region ES + sidecar-auth Secrets the example references +# (one ES Secret + one sidecar-auth Secret per region; see §6 for SealedSecrets/ESO/Vault). +# Replace every <…> with your real coordinates. +for r in us-east-1 eu-west-1 ap-south-1; do + kubectl create secret generic sc4es-es-$r \ + --from-literal=es-auth-method=basic \ + --from-literal=es-username='' \ + --from-literal=es-password='' + kubectl create secret generic sc4es-arrow-$r \ + --from-literal=arrow-bearer-token='' +done + +# 3. Install (REQUIRES the Pro/Enterprise license Secret above — without it the federation CrashLoops, see §7/§10). +helm install fed ./softclient4es-federation -f my-values.yaml +helm test fed # GetCatalogs → 3 catalogs +``` + +> The example's `values.yaml` references these Secrets by name (`license.secretName: sc4es-pro-license`, each region's `elasticsearch.credentialsSecretName: sc4es-es-` + `auth.credentialsSecretName: sc4es-arrow-`). **Create them BEFORE `helm install`** — a missing Secret is the #1 first-install failure (a loud, self-healing CrashLoop; §6/§10). The exact key names (`license-key`, `es-auth-method`/`es-username`/`es-password`, `arrow-bearer-token`) are the chart's expected Secret data keys (§6). See [`examples/three-region/README.md`](../examples/three-region/README.md) for the full per-region narrative (GDPR data-residency, SRE incident-triage, mixed-version migration). + +Each `sidecars[]` entry deploys one Deployment + one Service and registers one `arrow.flight.federation.servers.` entry (host = `..svc.cluster.local:32010`). Mixed ES versions are first-class — the chart selects `softclient4es{6,7,8,9}-arrow-flight-sql` per `elasticsearchVersion`. **Adding a cluster** = add a `sidecars[]` entry + `helm upgrade`; **removing** = delete the entry + `helm upgrade`. + +> ⚠️ **All-or-nothing readiness (SPOF).** With the default `federation.probes.useGrpc: true`, the federation is Ready only if **every** sidecar's backing ES is reachable — one down region pulls the whole federation out of its Service. For partial availability set `federation.probes.useGrpc: false` (TCP readiness on 32020); the federation then stays Ready and fails only the queries that touch the down region. See §9. + +## 4. The R2b-ready path + +```bash +cp examples/heterogeneous-ready/values.yaml my-values.yaml +``` + +This is the three-region topology **plus commented-out R2b `duckdb-attach` placeholders** (PostgreSQL / MySQL / Snowflake) that show how heterogeneous sources slot into the same `servers` map when R2b ships (Epic 25/26). **In R1 they stay commented** — `GetCatalogs` returns 3, not 6. R1 has no `values.yaml` key for `duckdb-attach` (the `servers` map is ConfigMap-rendered from `sidecars[]` Flight SQL servers only). **Uncommenting them later counts toward the license quota** (`clusterCount` includes attach servers) — 3 ES + 3 attach = 6 > Pro's 5 → Enterprise. + +See [`examples/heterogeneous-ready/README.md`](../examples/heterogeneous-ready/README.md) for the R2b preview narrative. + +## 5. Configuration reference + +> `values.yaml` field → HOCON path / env var. **Verify against the chart's own `values.yaml` and README; this table is the authoritative mapping.** + +**Federation** (image `softclient4es-federation`; HOCON root `arrow.flight.federation.*`): + +| `values.yaml` | HOCON | Env var | Default | +|---|---|---|---| +| `replicaCount` | — | — | 1 | +| `image.repository` / `image.tag` / `image.pullPolicy` | — | — | `docker.io/softnetwork/softclient4es-federation` / `appVersion` / `IfNotPresent` | +| `federation.maxMemory` | `max-memory` | `FEDERATION_MAX_MEMORY` | `512m` | +| `federation.queryTimeoutSeconds` | `query-timeout-seconds` | `FEDERATION_QUERY_TIMEOUT` | 30 | +| `federation.health.port` | `health.port` | `FEDERATION_HEALTH_PORT` | 32021 | +| `federation.health.probeTimeoutSeconds` | `health.probe-timeout-seconds` | `FEDERATION_HEALTH_PROBE_TIMEOUT` | 5 | +| `federation.duckdb.path` | `duckdb.path` | `FEDERATION_DUCKDB_PATH` | `:memory:` | +| `federation.upgradeUrl` | `upgrade-url` | `FEDERATION_UPGRADE_URL` | portal pricing URL | +| (`host`/`port` fixed by chart) | `host` / `port` | `FEDERATION_HOST` / `FEDERATION_PORT` | `0.0.0.0` / 32020 | +| `federation.probes.useGrpc` | — (probe wiring) | — | true | +| `federation.tls.{enabled,secretName}` | — (Ingress) | — | false | +| `ingress.{enabled,className,annotations,hosts,tls}` | — (Ingress) | — | false | +| `license.publicKeySecretName` (+ `license.publicKeyKey`) | — | `SOFTCLIENT4ES_LICENSE_PUBLIC_KEY` | `""` / `license-public-key` | + +**Sidecar** (per `sidecars[]`; image auto-selected by `elasticsearchVersion`; HOCON root `arrow.flight.*` + elasticsql core `elastic.credentials.*`): + +| `values.yaml` | HOCON / config | Env var | Default | +|---|---|---|---| +| `sidecars[].elasticsearchVersion` (`6\|7\|8\|9`) | — (image selection) | — | — | +| `sidecars[].name` | `servers.` key (RFC1123, unique) | — | — | +| `sidecars[].elasticsearch.url` (or `.scheme/.host/.port`) | `elastic.credentials.*` | `ELASTIC_SCHEME` / `ELASTIC_HOST` / `ELASTIC_PORT` | — (NO single ES-URL env) | +| `sidecars[].elasticsearch.credentialsSecretName` | `elastic.credentials.*` | `ELASTIC_AUTH_METHOD` / `ELASTIC_CREDENTIALS_{USERNAME,PASSWORD,API_KEY,BEARER_TOKEN}` | — | +| `sidecars[].arrow.batchSize` | `batch-size` | `ARROW_BATCH_SIZE` | 1000 | +| `sidecars[].arrow.queryTimeoutSeconds` | `query-timeout-seconds` | `ARROW_QUERY_TIMEOUT_SECONDS` | 120 | +| `sidecars[].arrow.maxMemory` | `join.max-memory` | `ARROW_JOIN_MAX_MEMORY` | `256m` | +| `sidecars[].auth.{method,credentialsSecretName}` | `auth.*` + federation `servers..credentials` | `ARROW_AUTH_{METHOD,USERNAME,PASSWORD,BEARER_TOKEN,API_KEY}` + `CONFIG_FORCE_*` | `none` | +| `sidecars[].default` | `servers..default` | — | false (≤1 true) | +| `sidecars[].alias` | `servers..alias` | — | `name` | +| `sidecars[].tls` | `servers..tls` (outgoing) | — | false | +| `sidecars[].replicaCount` / `.resources` / `.image.{repository,tag}` | — | — | 1 / — / version-default | + +**License & telemetry** (elasticsql `licensing`; HOCON root `softclient4es.*`): + +| `values.yaml` | HOCON | Env var | Default | +|---|---|---|---| +| `license.secretName` → `license-key` | `softclient4es.license.key` | `SOFTCLIENT4ES_LICENSE_KEY` | `""` (Community) | +| `license.secretName` → `api-key` | `softclient4es.license.api-key` | `SOFTCLIENT4ES_API_KEY` | `""` | +| `telemetry.enabled` | `softclient4es.telemetry.enabled` | `SOFTCLIENT4ES_TELEMETRY_ENABLED` | true | + +> **No env override** (NOT chart-exposed): `softclient4es.license.{connect-timeout,read-timeout,grace-period,cache-dir,refresh.enabled,refresh.interval}` and `softclient4es.license.telemetry.enabled` (the license-refresh-metrics switch — DISTINCT from the daily-ping `softclient4es.telemetry.enabled` above; do not confuse them). + +## 6. Secret management — choosing a backend + +The chart **never creates a `Secret`** — you create it (raw, SealedSecrets, External Secrets Operator, or Vault Agent Injector) and reference it by name. One Secret feeds **both** the sidecar (`ARROW_AUTH_*`, `ELASTIC_*`) and, for federation→sidecar auth, the federation (`CONFIG_FORCE_*` via Typesafe Config `override_with_env_vars`). The four backends are documented in depth in [`docs/secret-backends.md`](secret-backends.md), with ready-to-adapt manifests under [`examples/sealed-secrets/`](../examples/sealed-secrets/) and [`examples/external-secrets/`](../examples/external-secrets/). In short: + +- **Raw `kubectl create secret`** — simplest; fine for dev / GitOps with encrypted-at-rest etcd. Keys: `es-auth-method`/`es-username`/`es-password`/`es-api-key`/`es-bearer-token` (ES), `arrow-username`/`arrow-password`/`arrow-bearer-token`/`arrow-api-key` (sidecar auth), `license-key`/`api-key` (license), `tls.crt`/`tls.key` (TLS). +- **SealedSecrets** — commit an encrypted `SealedSecret` to Git; the in-cluster controller decrypts it. Re-seal per controller (certs are per-controller; `kubeseal` CLI version must match the controller `appVersion`, not the Helm chart version). +- **External Secrets Operator (ESO)** — sync from AWS/GCP Secret Manager or Vault. Examples use `external-secrets.io/v1` (the stable API; `v1beta1` was removed at ESO v0.17). +- **Vault Agent Injector** — inject secrets as files/env via pod annotations. + +> **RFC1123 sidecar names are mandatory.** The federation→sidecar credential injection mangles the sidecar `name` into a `CONFIG_FORCE_*` env path; a name with `_`/`.`/uppercase mangles wrong and the federation `sys.exit(1)`s. Use `[a-z0-9-]` names. **A wrong/missing Secret is a loud, SELF-HEALING CrashLoop** — the pod restarts and boots Ready once the Secret materializes (e.g. after ESO syncs). Do NOT uninstall; check the Secret's data keys. + +**TLS at the Ingress, not the pod.** The federation listener is plaintext-only (`Location.forGrpcInsecure`). `federation.tls.{enabled,secretName}` drives an **Ingress** `tls:` block (cert-manager `kubernetes.io/tls`); since Flight SQL is gRPC, the Ingress controller must proxy gRPC — nginx annotation `nginx.ingress.kubernetes.io/backend-protocol: "GRPC"` (NOT `GRPCS`; the pod upstream is plaintext h2c). This is DISTINCT from `sidecars[].tls` (outgoing per-downstream client TLS → `servers..tls`). In-cluster plaintext clients hit the Service (32020) directly. + +## 7. Licensing + +The federation reads its license from a referenced Secret as `SOFTCLIENT4ES_LICENSE_KEY` (offline Ed25519 JWT) and/or `SOFTCLIENT4ES_API_KEY` (automated portal provisioning). The gate is a **per-platform `maxClusters` quota**, enforced at startup against the number of `servers` (sidecars): + +| Tier | `maxClusters` | Federation clusters you can run | +|---|---|---| +| **Community** (no license) | 1 | single-cluster federation — **free** | +| **Pro** | 5 | up to 5 sidecars / clusters | +| **Enterprise** | unlimited | any | + +Exceeding the quota → the federation logs the over-quota error and `sys.exit(1)` → **CrashLoopBackOff by design** (see §10). Federation is NOT a paid feature — single-cluster is the free adoption tier; the quota is on *cluster count*. To opt out of the daily anonymous usage ping, set `telemetry.enabled: false` (→ `SOFTCLIENT4ES_TELEMETRY_ENABLED=false`); this has zero impact on functionality or your license. + +> **Offline verification.** For air-gapped or strict-egress clusters, mount the Ed25519 public JWK via `license.publicKeySecretName` (key `license.publicKeyKey`, default `license-public-key`) → `SOFTCLIENT4ES_LICENSE_PUBLIC_KEY`. This lets the federation verify a Pro/Enterprise JWT entirely offline (no portal round-trip). It is gated independently of `license.secretName`. + +## 8. ES-version mixing & migration + +Sidecars are independent — there is **no constraint** on mixing ES 6/7/8/9 in one federation at R1. The canonical migration story: run ES 8 sidecars in two regions while you migrate one region to ES 9; flip that region's `sidecars[].elasticsearchVersion` from `8` to `9` and `helm upgrade` once the ES cluster is upgraded. Cross-version JOINs with version-specific SQL features are the one caveat — flag them in testing (advanced cross-version cases surface in customer testing). + +## 9. Upgrades & rollback + +- **Upgrade:** edit `values.yaml`, `helm upgrade fed ./softclient4es-federation -f my-values.yaml`. Adding/removing a sidecar re-renders the federation ConfigMap; the federation pod restarts to pick up the new `servers` map. +- **Rollback:** `helm rollback fed ` (`helm history fed` to list). +- **Secret rotation** is start-time: after changing a Secret, `kubectl rollout restart deployment` for BOTH the sidecar AND the federation (one Secret feeds both — restart both or the two sides drift). +- **Probes:** native gRPC readiness needs K8s 1.27+. On clusters **older than 1.27**, set `federation.probes.useGrpc: false` (TCP readiness on 32020) — this is the recommended fallback and needs **no extra binary**. The `exec`+`grpc_health_probe` approach is NOT a drop-in: the `grpc_health_probe` binary is **not bundled in the federation image**, so you would have to bake it into a custom image (or an init-container copy) yourself before an `exec` probe can run it. For nearly all operators, `useGrpc: false` is the simpler and supported path below 1.27. + +## 10. Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Federation pod `CrashLoopBackOff`, log "exceeds maxClusters" / `sys.exit(1)` | More sidecars than the license tier allows (Community=1, Pro=5) | Add/upgrade the license Secret (Pro/Enterprise), or reduce `sidecars[]`. §7 | +| `CrashLoopBackOff` after uncommenting R2b `duckdb-attach` | `clusterCount` counts attach servers too → over Pro's 5 | Enterprise license, or keep R2b commented in R1. §4 | +| `CrashLoopBackOff`, "invalid credentials" at config load | `auth.method` ≠ none with creds only in a Secret but `federation.credentialsFromEnv` off, OR a non-RFC1123 sidecar name mangled the `CONFIG_FORCE_*` path | Enable `credentialsFromEnv`; rename sidecar to `[a-z0-9-]`. §6 | +| `CrashLoopBackOff` immediately at boot, no clear error | `readOnlyRootFilesystem` with no writable `/tmp` — the DuckDB JNI native lib can't extract | Ensure the `/tmp` `emptyDir` is present (chart default); don't override the volume. | +| Federation NotReady, "could not connect to sidecar X" | Sidecar pod / its backing ES unreachable; all-or-nothing gRPC readiness pulls the whole federation | Check sidecar pod + ES; for partial availability set `federation.probes.useGrpc: false`. §3/§9 | +| Wrong Secret keys / ESO sync lag | `optional:true` env can't validate Secret CONTENTS → boot-time `validate()` CrashLoop | Fix the Secret's data keys; the pod self-heals on the next restart — do NOT uninstall. §6 | +| License validation failure (**Pro/Enterprise only**) | A license JWT WAS supplied but is malformed / clock-skewed / expired beyond the 14-day grace | Re-provision via the portal (`SOFTCLIENT4ES_API_KEY`); check the JWT and node clock. **N.B.** with NO license (blank `SOFTCLIENT4ES_LICENSE_KEY`) the federation does NOT error — it runs on Community (1-cluster quota); that fallback is expected, not a failure. Community has no API-key/refresh path, so the portal re-provision step applies ONLY to a paid license. | +| TLS cert renewal failure | Cert lives at the **Ingress** (federation pod is plaintext) | Renew the cert-manager Certificate / Ingress `tls` Secret; nginx needs `backend-protocol: "GRPC"`. §6 (TLS) | +| ES connection failure from a sidecar | Wrong `ELASTIC_SCHEME/HOST/PORT/AUTH_METHOD` or ES creds | Verify the ES Secret + reachability; remember there is NO single ES-URL — scheme/host/port are separate. §5 | +| ES-version mismatch | `elasticsearchVersion` doesn't match the backing ES major | Set the sidecar's `elasticsearchVersion` to the ES cluster's major. §8 | + +### SRE incident-triage walkthrough (three-region) + +1. **Alert:** dashboards/queries failing across regions. `kubectl get pods` → the `fed-softclient4es-federation` pod is `0/1 NotReady`. +2. **Confirm the SPOF:** `kubectl describe pod` shows the gRPC readiness probe failing. With `useGrpc:true`, one unreachable region makes the *whole* federation NotReady (all-or-nothing aggregate). +3. **Find the bad region:** `kubectl get pods -l app.kubernetes.io/component=sidecar` → identify the `NotReady` sidecar; `kubectl logs` it for the ES connection error. +4. **Mitigate fast:** re-apply your existing values file AND flip the probe — `helm upgrade fed ./softclient4es-federation -f my-values.yaml --set federation.probes.useGrpc=false` → the federation goes Ready and serves the two healthy regions; only queries touching the down region fail. (Per-alias readiness is an R1.x backlog item.) ⚠️ **Always include `-f my-values.yaml`**: a bare `helm upgrade … --set …` resets every other value to the chart default (Helm does NOT reuse the previous release's values unless you pass `-f` or `--reuse-values`), which would silently revert your three-region `sidecars[]` / license / topology mid-incident. +5. **Fix root cause:** restore the down region's ES / sidecar, then flip `useGrpc` back to `true` for fail-closed semantics. + +## 11. Performance tuning *(placeholder — iterate after R1 telemetry)* + +> Hard numbers land after R1 telemetry. The qualitative levers today: + +- **Scale federation replicas** (`replicaCount`) for concurrent-query throughput; the federation is stateless per query (DuckDB `:memory:`). +- **Scale sidecars** (`sidecars[].replicaCount`, document HA at 3) for per-cluster availability and parallelism. +- **`federation.maxMemory` / `sidecars[].arrow.maxMemory`** size the DuckDB / JOIN engine; raise for large cross-cluster JOINs, watch pod memory limits. +- **`batchSize`** trades latency vs throughput on streaming. +- Resource recommendations per cluster-count tier: TBD — seeded after R1 telemetry. + +## 12. SLA implications + +Pro includes 48h support. This guide is intended to be authoritative enough that the common scenarios (install, add/remove a cluster, mixed-version migration, the over-quota and connectivity failure modes) are self-serviceable without a ticket. + +## For Epic 17 + +Epic 17 (Documentation & Marketing) consumes, for R1 launch-day docs: **this operator guide** (linked/included from the launch docs), the **`examples/` topology folders** ([single-cluster](../examples/single-cluster/README.md), [three-region](../examples/three-region/README.md), [heterogeneous-ready](../examples/heterogeneous-ready/README.md) — each with README + ASCII diagram), and the **SRE incident-triage walkthrough** (§10, linked from the R1 marketing SRE-wedge story). diff --git a/softclient4es-federation/docs/secret-backends.md b/softclient4es-federation/docs/secret-backends.md new file mode 100644 index 0000000..a0136cb --- /dev/null +++ b/softclient4es-federation/docs/secret-backends.md @@ -0,0 +1,42 @@ +# Choosing a secret backend + +The SoftClient4ES federation chart **references** Kubernetes Secrets by name but never +creates them — so you bring your own secret-management workflow. The chart expects each +Secret to carry specific data keys (see the README "Secret key-name contract" table); the +four common ways to populate them are below. The chart prescribes none — pick what fits +your platform and compliance posture. + +## Raw Kubernetes Secret + +The simplest path: `kubectl create secret generic es-prod-us --from-literal=es-username=… +--from-literal=es-password=…`. The Secret lives in etcd (enable etcd encryption-at-rest). +Best for clusters where Secret material is managed out-of-band (CI, a vault export) and you +accept GitOps storing only references, not values. +Docs: https://kubernetes.io/docs/concepts/configuration/secret/ + +## Sealed Secrets (Bitnami) + +Encrypt a Secret with the cluster's public cert so the *SealedSecret* (safe to commit to Git) +is decrypted only by the in-cluster controller: `kubeseal`. Good for GitOps without an external +secret store. Pin the `kubeseal` CLI to the controller's appVersion (the Helm chart version is a +different number). Re-seal per cluster (certs differ) and per namespace (strict-scoped by +default). See `examples/sealed-secrets/`. +Docs: https://github.com/bitnami-labs/sealed-secrets + +## External Secrets Operator (ESO) + +Sync secrets from AWS Secrets Manager / GCP Secret Manager / Azure Key Vault / HashiCorp Vault +into native K8s Secrets via an `ExternalSecret` + `SecretStore`. The cloud store is the source of +truth; ESO refreshes on an interval. Best when secrets already live in a managed store and you +want rotation. Map remote keys to the chart-expected data keys. Use the stable +`external-secrets.io/v1` API (`v1beta1` was removed at ESO v0.17.0). See +`examples/external-secrets/`. +Docs: https://external-secrets.io + +## Vault Agent Injector + +Inject secrets as files into the Pod via a Vault sidecar + pod annotations (no K8s Secret +object). Strongest isolation (secrets never touch etcd) but needs Vault + a custom container +command to read the injected file into the env the chart expects — pair with `useEnvFrom: false` +and an entrypoint shim. Best for Vault-centric orgs with strict no-etcd-secrets policies. +Docs: https://developer.hashicorp.com/vault/docs/platform/k8s/injector diff --git a/softclient4es-federation/examples/external-secrets/README.md b/softclient4es-federation/examples/external-secrets/README.md new file mode 100644 index 0000000..10572eb --- /dev/null +++ b/softclient4es-federation/examples/external-secrets/README.md @@ -0,0 +1,39 @@ +# External Secrets Operator (ESO) examples (Story 16.3) + +These manifests sync secrets from a managed store (AWS Secrets Manager / GCP Secret Manager / +HashiCorp Vault) into the native Kubernetes Secrets the chart references, via +[External Secrets Operator](https://external-secrets.io). The cloud store is the source of +truth; ESO refreshes on an interval. The chart never creates a Secret — ESO does. + +| File | Backend | Materializes Secret | Chart reference | +|---|---|---|---| +| `aws-secrets-manager.externalsecret.yaml` | AWS Secrets Manager | `es-prod-us` | `sidecars[].elasticsearch.credentialsSecretName` | +| `gcp-secret-manager.externalsecret.yaml` | GCP Secret Manager | `prod-us-arrow-auth` | `sidecars[].auth.credentialsSecretName` (feeds BOTH sides) | +| `vault.externalsecret.yaml` | HashiCorp Vault (KV v2) | `es-prod-us` | `sidecars[].elasticsearch.credentialsSecretName` | + +## API version + +All examples use **`external-secrets.io/v1`** — the STABLE ESO API. `v1beta1` was deprecated +then **removed at ESO v0.17.0** (the webhook auto-converts `v1beta1` → `v1` from v0.16.x), so a +`v1beta1` example fails `kubectl apply` on a current ESO. On ESO **< 0.16** substitute +`v1beta1`. + +## Map remote keys to the chart-expected keys + +Each `ExternalSecret.spec.data[].secretKey` MUST be a chart-expected data key (see the chart +README "Secret key-name contract"), e.g. `es-username`, `es-password`, `arrow-bearer-token`. +Point `remoteRef` at wherever the value lives in your managed store. + +## CI / validation note + +ESO CRDs are **not** in the default kubeconform schema set, so these examples are +schema-validated only when an ESO `-schema-location` is supplied (otherwise skipped, not +failed). A **live** ESO sync needs a real cloud store / Vault, so the ESO + Vault paths are the +**documented best-effort** tier (the raw-Secret and SealedSecrets paths are the CI-tested tier — +see Story 16.5). + +## Vault Agent Injector (alternative) + +The `vault.externalsecret.yaml` here uses ESO's Vault provider. The Vault **Agent Injector** +(sidecar injection via pod annotations) is a different integration — it injects a file, not a +K8s Secret — covered in `../../docs/secret-backends.md`. diff --git a/softclient4es-federation/examples/external-secrets/aws-secrets-manager.externalsecret.yaml b/softclient4es-federation/examples/external-secrets/aws-secrets-manager.externalsecret.yaml new file mode 100644 index 0000000..ce85050 --- /dev/null +++ b/softclient4es-federation/examples/external-secrets/aws-secrets-manager.externalsecret.yaml @@ -0,0 +1,52 @@ +# External Secrets Operator (ESO) — AWS Secrets Manager backend. +# Materializes the chart-expected Secret `es-prod-us` from an AWS Secrets Manager secret. +# Prereq: ESO installed; IRSA/role with secretsmanager:GetSecretValue. The chart does NOT +# create the K8s Secret — ESO does, from your cloud secret store. +# +# API VERSION: uses `external-secrets.io/v1` — the STABLE API. ESO DEPRECATED and then REMOVED +# `v1beta1` at v0.17.0 (the webhook auto-converts v1beta1->v1 from v0.16.x). If you run ESO < 0.16 +# substitute `v1beta1`; on ESO >= 0.17 only `v1` is served. Validate the example against YOUR +# installed ESO CRDs (kubeconform with -schema-location pointed at the ESO CRD schemas — ESO CRDs +# are not in the default kubeconform schema set, so these examples are schema-checked only when +# the ESO schema location is provided, else skipped, not failed). +apiVersion: external-secrets.io/v1 +kind: SecretStore +metadata: + name: aws-secretsmanager + namespace: default +spec: + provider: + aws: + service: SecretsManager + region: us-east-1 + auth: + jwt: + serviceAccountRef: + name: external-secrets-sa +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: es-prod-us + namespace: default +spec: + refreshInterval: 1h + secretStoreRef: + name: aws-secretsmanager + kind: SecretStore + target: + name: es-prod-us # MUST match sidecars[].elasticsearch.credentialsSecretName + creationPolicy: Owner + data: + - secretKey: es-auth-method # chart-expected key (FACT C contract) + remoteRef: + key: prod/es-prod-us + property: auth_method + - secretKey: es-username + remoteRef: + key: prod/es-prod-us + property: username + - secretKey: es-password + remoteRef: + key: prod/es-prod-us + property: password diff --git a/softclient4es-federation/examples/external-secrets/gcp-secret-manager.externalsecret.yaml b/softclient4es-federation/examples/external-secrets/gcp-secret-manager.externalsecret.yaml new file mode 100644 index 0000000..9d35fbe --- /dev/null +++ b/softclient4es-federation/examples/external-secrets/gcp-secret-manager.externalsecret.yaml @@ -0,0 +1,39 @@ +# External Secrets Operator (ESO) — GCP Secret Manager backend. +# Materializes the chart-expected Secret `prod-us-arrow-auth` (feeds BOTH the sidecar's +# ARROW_AUTH_* AND the federation's CONFIG_FORCE_* — Story 16.3, FACT A) from GCP Secret +# Manager. Prereq: ESO installed; Workload Identity bound to a GCP SA with +# secretmanager.versions.access. See aws-secrets-manager.externalsecret.yaml for the +# `external-secrets.io/v1` API-version note. +apiVersion: external-secrets.io/v1 +kind: SecretStore +metadata: + name: gcp-secretmanager + namespace: default +spec: + provider: + gcpsm: + projectID: my-gcp-project + auth: + workloadIdentity: + clusterLocation: us-central1 + clusterName: my-cluster + serviceAccountRef: + name: external-secrets-sa +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: prod-us-arrow-auth + namespace: default +spec: + refreshInterval: 1h + secretStoreRef: + name: gcp-secretmanager + kind: SecretStore + target: + name: prod-us-arrow-auth # MUST match sidecars[].auth.credentialsSecretName + creationPolicy: Owner + data: + - secretKey: arrow-bearer-token # chart-expected key (FACT C contract) + remoteRef: + key: prod-us-arrow-token diff --git a/softclient4es-federation/examples/external-secrets/vault.externalsecret.yaml b/softclient4es-federation/examples/external-secrets/vault.externalsecret.yaml new file mode 100644 index 0000000..30769dd --- /dev/null +++ b/softclient4es-federation/examples/external-secrets/vault.externalsecret.yaml @@ -0,0 +1,44 @@ +# External Secrets Operator (ESO) — HashiCorp Vault (KV v2) backend. +# NB: this is the ESO->Vault path (a SecretStore with a vault provider). The Vault Agent +# Injector ALTERNATIVE (sidecar-injection via pod annotations) is a DIFFERENT integration +# documented in docs/secret-backends.md — it injects a file, not a K8s Secret, so it pairs +# with `useEnvFrom: false` + a custom command, and is the "best-effort" tier. +# See aws-secrets-manager.externalsecret.yaml for the `external-secrets.io/v1` API-version note. +apiVersion: external-secrets.io/v1 +kind: SecretStore +metadata: + name: vault-backend + namespace: default +spec: + provider: + vault: + server: "https://vault.example.com:8200" + path: "secret" + version: "v2" + auth: + kubernetes: + mountPath: "kubernetes" + role: "softclient4es" +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: es-prod-us + namespace: default +spec: + refreshInterval: 1h + secretStoreRef: + name: vault-backend + kind: SecretStore + target: + name: es-prod-us # MUST match sidecars[].elasticsearch.credentialsSecretName + creationPolicy: Owner + data: + - secretKey: es-username + remoteRef: + key: prod/es-prod-us + property: username + - secretKey: es-password + remoteRef: + key: prod/es-prod-us + property: password diff --git a/softclient4es-federation/examples/heterogeneous-ready/README.md b/softclient4es-federation/examples/heterogeneous-ready/README.md new file mode 100644 index 0000000..9bbb767 --- /dev/null +++ b/softclient4es-federation/examples/heterogeneous-ready/README.md @@ -0,0 +1,74 @@ +# Heterogeneous-ready federation (R1 today → R2b tomorrow) — Pro / Enterprise + +Runs the **three-region ES topology TODAY on R1** (3 Elasticsearch sidecars), and is +laid out so the **R2b heterogeneous sources** (PostgreSQL, MySQL, Snowflake, …) slot +into the **same federation `servers` map** when R2b ships — **no restructuring** of +this deployment. + +## What works today (R1) + +Exactly the three-region topology: a Flight SQL endpoint federating 3 ES clusters +(us-east-1 ES 8, eu-west-1 ES 8, ap-south-1 ES 9), with cross-cluster JOIN. Needs a +**Pro/Enterprise license** (3 clusters > Community's `maxClusters=1`) — see the +quota table in `../three-region/README.md`. + +## What slots in tomorrow (R2b) + +The federation `servers` map already supports `type = "duckdb-attach"` entries +(PostgreSQL / MySQL / Snowflake / … via DuckDB ATTACH). When R2b ships, those join +the SAME map your ES sidecars register into — a relational source becomes a federated +catalog you can `JOIN` against your ES indices, with no change to the federation +deployment topology. The `values.yaml` carries a commented preview of the exact HOCON. + +> ⚠️⚠️ **The R2b preview in `values.yaml` is INERT on R1 — uncommenting it does NOTHING.** +> R1 has **no `values.yaml` key** that parses `duckdb-attach` (or a raw `servers { }` +> HOCON map). The federation `servers` map is rendered into a ConfigMap **entirely from +> the `sidecars[]` array** (Flight SQL sidecars only). If you uncomment the preview block +> and `helm install`, Helm discards it as comments, the chart ignores it, no relational +> catalog appears, and `SHOW CATALOGS` still returns **3** (the ES sidecars). The block is +> a documentation illustration of the FUTURE R2b shape — not an activation switch. R2b +> (Epic 25/26) is what makes these sources live. + +> ⚠️ **Quota note:** each active `duckdb-attach` server ALSO counts toward `maxClusters` +> (`clusterCount = servers.size`). 3 ES + 3 attach = 6 clusters would exceed Pro's +> `maxClusters=5` → you'd need Enterprise. (A smaller mix still fits Pro: 3 ES + 2 attach +> = 5 clusters is exactly Pro's cap; only the 6th server tips you into Enterprise.) Plan +> your tier before activating R2b sources. + +## Topology + +``` + ┌──────────────────────────────────┐ + Flight SQL client ───┼─► Federation (Flight SQL :32020) │ + └───┬────────┬────────┬─────────────┘ + R1-ACTIVE │ │ │ + ┌─────────────▼┐ ┌─────▼──────┐ ┌▼────────────┐ + │ us-east-1 ES8│ │ eu-west-1 │ │ ap-south-1 │ + │ sidecar │ │ ES8 sidecar│ │ ES9 sidecar │ + └──────────────┘ └────────────┘ └─────────────┘ + + R2b-PREVIEW (commented; slot into the SAME servers map) + ┌╌╌╌╌╌╌╌╌╌╌╌╌╌┐ ┌╌╌╌╌╌╌╌╌╌╌╌╌┐ ┌╌╌╌╌╌╌╌╌╌╌╌╌╌┐ + ┊ analytics_pg ┊ ┊ orders_mysql┊ ┊ snowflake_wh ┊ + ┊ (PostgreSQL) ┊ ┊ (MySQL) ┊ ┊ (Snowflake) ┊ + └╌╌╌╌╌╌╌╌╌╌╌╌╌┘ └╌╌╌╌╌╌╌╌╌╌╌╌┘ └╌╌╌╌╌╌╌╌╌╌╌╌╌┘ + (duckdb-attach — Epic 25/26) +``` + +## Install (R1) + +Same as three-region (Pro license + per-region Secrets), then: + +```bash +helm install softclient4es-federation softclient4es-federation \ + -f examples/heterogeneous-ready/values.yaml +``` + +## Verify + +```bash +helm test softclient4es-federation # SHOW CATALOGS returns 3 (the active ES clusters) +``` + +The R2b `duckdb-attach` placeholders are **commented out** — they do not register +and do not count toward `SHOW CATALOGS` (which returns **3**) until R2b activates them. diff --git a/softclient4es-federation/examples/heterogeneous-ready/values.yaml b/softclient4es-federation/examples/heterogeneous-ready/values.yaml new file mode 100644 index 0000000..a02913c --- /dev/null +++ b/softclient4es-federation/examples/heterogeneous-ready/values.yaml @@ -0,0 +1,108 @@ +# ───────────────────────────────────────────────────────────────────────────── +# SoftClient4ES Federation — HETEROGENEOUS-READY example. +# +# This is the THREE-REGION topology (3 ES sidecars) you can run on R1 TODAY, plus a +# commented-out preview of the R2b "duckdb-attach" heterogeneous sources (PostgreSQL, +# MySQL, Snowflake). When R2b ships, those slot into the SAME federation `servers` +# map WITHOUT restructuring this deployment — the federation already federates them +# the moment they are activated. +# +# ⚠️ Like three-region, this needs a PRO/ENTERPRISE license (3 active ES sidecars = +# 3 clusters; Community maxClusters=1). See README. +# +# Install: +# helm install softclient4es-federation softclient4es-federation \ +# -f examples/heterogeneous-ready/values.yaml +# ───────────────────────────────────────────────────────────────────────────── + +license: + secretName: "sc4es-pro-license" + +telemetry: + enabled: true + +federation: + probes: + useGrpc: true + +# R1-ACTIVE: 3 Elasticsearch sidecars (same as three-region). +sidecars: + - name: us-east-1 + elasticsearchVersion: 8 + elasticsearch: + url: "https://es-us-east-1.example.com:9200" + credentialsSecretName: "sc4es-es-us-east-1" + auth: + method: bearer + credentialsSecretName: "sc4es-arrow-us-east-1" + default: true + - name: eu-west-1 + elasticsearchVersion: 8 + elasticsearch: + url: "https://es-eu-west-1.example.com:9200" + credentialsSecretName: "sc4es-es-eu-west-1" + auth: + method: bearer + credentialsSecretName: "sc4es-arrow-eu-west-1" + - name: ap-south-1 + elasticsearchVersion: 9 + elasticsearch: + url: "https://es-ap-south-1.example.com:9200" + credentialsSecretName: "sc4es-es-ap-south-1" + auth: + method: bearer + credentialsSecretName: "sc4es-arrow-ap-south-1" + +# ───────────────────────────────────────────────────────────────────────────── +# R2b PREVIEW — heterogeneous "duckdb-attach" sources (NOT active in R1). +# +# ⚠️⚠️ UNCOMMENTING THE BLOCK BELOW DOES NOTHING ON R1. ⚠️⚠️ +# This is NOT a chart values key. There is NO `values.yaml` key in R1 that parses +# `duckdb-attach` / a raw `servers { }` HOCON map. The federation `servers` map is +# rendered into a ConfigMap ENTIRELY from the `sidecars[]` array above (Flight SQL +# servers only — one entry per `elasticsearchVersion`). If you uncomment the HOCON +# below and `helm install`, Helm treats it as inert YAML comments, the chart ignores +# it, the federation never sees a PostgreSQL/MySQL/Snowflake catalog, and +# `SHOW CATALOGS` still returns 3 (the ES sidecars only). Do NOT file a "my PG catalog +# didn't show up" ticket — R2b (Epic 25/26) is what wires these into the `servers` map. +# +# The entries below show the EXACT HOCON shape these sources will take in the federation +# `servers` map once R2b ships — copied verbatim from +# federation/src/main/resources/reference.conf. They are illustrative ONLY here. +# +# ⚠️ Each duckdb-attach server, once ACTIVE, counts toward maxClusters too +# (clusterCount = servers.size, ALL server types). 3 ES + 3 attach = 6 clusters → +# exceeds Pro's maxClusters=5 → needs Enterprise. (3 ES + 2 attach = 5 still fits +# Pro exactly; only the 6th server tips into Enterprise.) Plan tier accordingly. +# +# ── DuckDB ATTACH (PostgreSQL) ── +# analytics_pg { +# host = "pg.internal" +# port = 5432 +# type = "duckdb-attach" +# dialect = "postgresql" +# read-only = true +# expected-catalog = "analytics_db" +# credentials { username = "readonly" password = "secret" } +# } +# ── DuckDB ATTACH (MySQL) ── +# orders_mysql { +# host = "mysql.internal" +# port = 3306 +# type = "duckdb-attach" +# dialect = "mysql" +# expected-catalog = "orders_db" +# credentials { username = "reader" password = "secret" } +# } +# ── DuckDB ATTACH (Snowflake — custom attach-string + secret) ── +# snowflake_wh { +# host = "" +# type = "duckdb-attach" +# dialect = "snowflake" +# read-only = true +# duckdb { +# attach-string = "account=my-account" +# secret { TYPE = "snowflake" ACCOUNT = "my-account" USER = "admin" PASSWORD = "pw" } +# } +# } +# ───────────────────────────────────────────────────────────────────────────── diff --git a/softclient4es-federation/examples/sealed-secrets/README.md b/softclient4es-federation/examples/sealed-secrets/README.md new file mode 100644 index 0000000..1c6c992 --- /dev/null +++ b/softclient4es-federation/examples/sealed-secrets/README.md @@ -0,0 +1,53 @@ +# Sealed Secrets examples (Story 16.3) + +These two `SealedSecret` manifests show the **shape** the chart expects for a +GitOps-friendly secret workflow with [Bitnami Sealed Secrets](https://github.com/bitnami-labs/sealed-secrets). + +> ⚠️ **They are NON-FUNCTIONAL placeholders.** The `encryptedData` was sealed with an +> example key and **will not decrypt** in your cluster. You **must re-seal** every +> SealedSecret with **your** controller's public cert. SealedSecrets are sealed +> per-controller and (by default) per-namespace. + +| File | Materializes Secret | Chart reference | +|---|---|---| +| `es-credentials.sealedsecret.yaml` | `es-prod-us` (keys `es-auth-method`, `es-username`, `es-password`) | `sidecars[].elasticsearch.credentialsSecretName` | +| `sidecar-auth.sealedsecret.yaml` | `prod-us-arrow-auth` (key `arrow-bearer-token`) | `sidecars[].auth.credentialsSecretName` (feeds BOTH sides) | + +## Re-seal with your controller + +```sh +# 1. Install the controller (repo MOVED bitnami-labs -> bitnami): +helm repo add sealed-secrets https://bitnami.github.io/sealed-secrets +helm upgrade --install sealed-secrets sealed-secrets/sealed-secrets -n kube-system + +# 2. Re-seal the ES credentials Secret for YOUR cluster + namespace: +kubectl create secret generic es-prod-us --namespace \ + --from-literal=es-auth-method=basic \ + --from-literal=es-username=elastic \ + --from-literal=es-password='' \ + --dry-run=client -o yaml \ +| kubeseal --controller-name sealed-secrets --controller-namespace kube-system \ + --format yaml > es-credentials.sealedsecret.yaml + +# 3. Re-seal the sidecar/federation auth Secret (ONE Secret feeds both sides): +kubectl create secret generic prod-us-arrow-auth --namespace \ + --from-literal=arrow-bearer-token='' \ + --dry-run=client -o yaml \ +| kubeseal --controller-name sealed-secrets --controller-namespace kube-system \ + --format yaml > sidecar-auth.sealedsecret.yaml + +kubectl apply -f es-credentials.sealedsecret.yaml -f sidecar-auth.sealedsecret.yaml +``` + +## Gotchas + +- **`kubeseal` CLI version ≠ Helm chart version.** Chart `2.16.2` bundles controller/CLI + appVersion `0.27.2`. Building the CLI URL from the chart version 404s. Pin the CLI to the + controller's **appVersion**. +- **Namespace scope.** A strict-scoped SealedSecret is bound to an exact `name + namespace`. + Deploying the chart into a different namespace requires re-sealing with `--namespace `, + or sealing `--scope namespace-wide` / `--scope cluster-wide` with the matching + `sealedsecrets.bitnami.com/namespace-wide` / `cluster-wide` annotation on the source Secret. +- **A wrong namespace fails SILENTLY** — the Secret never materializes, the chart's + `optional: true` `secretKeyRef`s no-op, and the federation CrashLoops at boot with a + `FlightCredentials`/`validate()` error. See the chart README "Secrets, TLS & Ingress". diff --git a/softclient4es-federation/examples/sealed-secrets/es-credentials.sealedsecret.yaml b/softclient4es-federation/examples/sealed-secrets/es-credentials.sealedsecret.yaml new file mode 100644 index 0000000..2053e72 --- /dev/null +++ b/softclient4es-federation/examples/sealed-secrets/es-credentials.sealedsecret.yaml @@ -0,0 +1,46 @@ +# EXAMPLE ONLY — NON-FUNCTIONAL PLACEHOLDER. +# The encryptedData below was sealed with an EXAMPLE public key and WILL NOT decrypt +# in your cluster. You MUST re-seal with YOUR sealed-secrets controller's cert: +# +# kubectl create secret generic es-prod-us \ +# --from-literal=es-auth-method=basic \ +# --from-literal=es-username=elastic \ +# --from-literal=es-password='' \ +# --dry-run=client -o yaml \ +# | kubeseal --controller-name sealed-secrets --controller-namespace kube-system \ +# --format yaml > es-credentials.sealedsecret.yaml +# +# GOTCHA 1 (CLI vs chart version): the kubeseal CLI version must match the controller +# appVersion (the Helm CHART version is a DIFFERENT number — chart 2.16.2 bundles +# controller/CLI 0.27.2; the repo MOVED bitnami-labs.github.io -> bitnami.github.io). A +# mismatched CLI can produce a SealedSecret the controller can't unseal. Pin the CLI to the +# controller's appVersion. +# GOTCHA 2 — SCOPE/NAMESPACE: by DEFAULT a SealedSecret is *strict*-scoped — sealed for an +# EXACT name + namespace. This placeholder is sealed for name=es-prod-us, namespace=default. +# If you deploy the chart into a DIFFERENT namespace (e.g. --namespace federation), you MUST +# re-seal with --namespace on the `kubectl create secret` (the namespace is part of +# the encryption AAD), OR seal namespace-/cluster-wide: add `--scope namespace-wide` (any name, +# fixed ns) or `--scope cluster-wide` (any name, any ns) to kubeseal AND the matching +# `sealedsecrets.bitnami.com/namespace-wide`/`cluster-wide` annotation on the source Secret. +# A name/namespace mismatch fails to decrypt SILENTLY (the Secret is never materialized -> +# the chart's optional:true secretKeyRefs no-op -> CrashLoop). Re-seal (strict, default ns): +# kubectl create secret generic es-prod-us --namespace \ +# --from-literal=es-auth-method=basic --from-literal=es-username=elastic \ +# --from-literal=es-password='' --dry-run=client -o yaml \ +# | kubeseal --controller-name sealed-secrets --controller-namespace kube-system --format yaml \ +# > es-credentials.sealedsecret.yaml +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: es-prod-us + namespace: default +spec: + encryptedData: + es-auth-method: AgBPLACEHOLDERNOTREALRESEALREQUIRED== + es-username: AgCPLACEHOLDERNOTREALRESEALREQUIRED== + es-password: AgDPLACEHOLDERNOTREALRESEALREQUIRED== + template: + metadata: + name: es-prod-us + namespace: default + type: Opaque diff --git a/softclient4es-federation/examples/sealed-secrets/sidecar-auth.sealedsecret.yaml b/softclient4es-federation/examples/sealed-secrets/sidecar-auth.sealedsecret.yaml new file mode 100644 index 0000000..0ca199c --- /dev/null +++ b/softclient4es-federation/examples/sealed-secrets/sidecar-auth.sealedsecret.yaml @@ -0,0 +1,20 @@ +# EXAMPLE ONLY — NON-FUNCTIONAL PLACEHOLDER (re-seal as in es-credentials.sealedsecret.yaml). +# Feeds BOTH the sidecar's ARROW_AUTH_* AND the federation's CONFIG_FORCE_* (one Secret — +# Story 16.3, FACT A). Re-seal with YOUR controller's cert: +# kubectl create secret generic prod-us-arrow-auth \ +# --from-literal=arrow-bearer-token='' \ +# --dry-run=client -o yaml | kubeseal --format yaml > sidecar-auth.sealedsecret.yaml +# Same namespace/scope gotcha as es-credentials.sealedsecret.yaml applies. +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: prod-us-arrow-auth + namespace: default +spec: + encryptedData: + arrow-bearer-token: AgXPLACEHOLDERNOTREALRESEALREQUIRED== + template: + metadata: + name: prod-us-arrow-auth + namespace: default + type: Opaque diff --git a/softclient4es-federation/examples/single-cluster/README.md b/softclient4es-federation/examples/single-cluster/README.md new file mode 100644 index 0000000..2614d8f --- /dev/null +++ b/softclient4es-federation/examples/single-cluster/README.md @@ -0,0 +1,70 @@ +# Single-cluster federation (FREE tier) + +Federation Flight SQL server in front of **one** Elasticsearch cluster, via one +Arrow Flight SQL sidecar. Runs on the **Community license — no license key needed** +(the cluster quota is `maxClusters=1`, and one sidecar fits). + +## When to use this + +- You have ONE Elasticsearch cluster and want a **Flight SQL endpoint** for it + (BI tools, ADBC, JDBC-over-Flight) with **cross-index JOIN routing** and licensing, + exposed externally — without standing up the federation + sidecar HOCON + probes by hand. +- You want to **start free** and grow into multi-region later: add a second `sidecars[]` + entry + a Pro license and you have the three-region topology (see `../three-region/`). + +## Topology + +``` + ┌─────────────────────────────┐ + Flight SQL client │ Federation (Flight SQL) │ + (BI / ADBC / JDBC) ─┼─► :32020 softclient4es- │ + │ federation │ + │ health gRPC :32021 │ + └──────────────┬──────────────┘ + │ in-cluster gRPC (plaintext) + ▼ + ┌─────────────────────────────┐ + │ Sidecar: primary (ES 8) │ + │ Arrow Flight SQL :32010 │ + └──────────────┬──────────────┘ + │ ELASTIC_SCHEME/HOST/PORT + ▼ + ┌─────────────────────────────┐ + │ Elasticsearch (1 cluster) │ + │ es.example.com:9200 │ + └─────────────────────────────┘ +``` + +## Install + +```bash +# Create the ES credentials Secret (skip for a noauth ES): +kubectl create secret generic sc4es-es-credentials \ + --from-literal=es-auth-method=basic \ + --from-literal=es-username=elastic \ + --from-literal=es-password='' + +helm install softclient4es-federation softclient4es-federation \ + -f examples/single-cluster/values.yaml \ + --set sidecars[0].elasticsearch.url=https://YOUR-ES:9200 +``` + +> **`elasticsearch.url` must be `scheme://host:port`.** The chart decomposes it into +> `ELASTIC_SCHEME`/`ELASTIC_HOST`/`ELASTIC_PORT` (there is no single ES-URL env). A +> scheme-less value defaults to `http`; a port-less value defaults to `9200`. For a +> TLS or non-9200 cluster, always include `https://` and the explicit port, or set +> `elasticsearch.scheme`/`.host`/`.port` directly (explicit fields win over `url`). + +## Verify + +```bash +helm test softclient4es-federation # SHOW CATALOGS returns 1 (the `primary` cluster) +``` + +`SHOW CATALOGS` (or ADBC `get_objects(depth="catalogs")`) returns **1** catalog. + +## Licensing + +**No license required.** A single-cluster federation runs on Community +(`maxClusters=1`). The moment you add a **second** sidecar you need a Pro/Enterprise +license — see `../three-region/README.md`. diff --git a/softclient4es-federation/examples/single-cluster/values.yaml b/softclient4es-federation/examples/single-cluster/values.yaml new file mode 100644 index 0000000..03a46c9 --- /dev/null +++ b/softclient4es-federation/examples/single-cluster/values.yaml @@ -0,0 +1,48 @@ +# ───────────────────────────────────────────────────────────────────────────── +# SoftClient4ES Federation — SINGLE-CLUSTER example. +# +# Federation Flight SQL server + ONE Arrow Flight SQL sidecar in front of ONE +# Elasticsearch cluster. This is the FREE tier: a single sidecar boots on the +# Community license (no license.secretName needed — maxClusters=1 permits exactly +# one cluster). Use it to expose JOIN routing + licensing + a Flight SQL endpoint +# for ONE ES cluster, externally reachable, with zero licensing cost. +# +# Install: +# helm install softclient4es-federation softclient4es-federation \ +# -f examples/single-cluster/values.yaml +# +# EDIT BEFORE INSTALL: sidecars[0].elasticsearch.url and (if your ES needs auth) +# create a Secret named `sc4es-es-credentials` and reference it below. +# ───────────────────────────────────────────────────────────────────────────── + +# No license needed — a single-cluster federation runs on Community (maxClusters=1). +license: + secretName: "" + +# Daily product-instance telemetry ping is ON by default. Set false to opt out. +telemetry: + enabled: true + +# Single downstream sidecar (no SPOF amplification → gRPC readiness default is fine). +sidecars: + - name: primary # RFC1123; servers.primary + k8s suffix + elasticsearchVersion: 8 # 6|7|8|9 → selects the sidecar image + elasticsearch: + # REPLACE with your ES endpoint. The chart DECOMPOSES this into + # ELASTIC_SCHEME/ELASTIC_HOST/ELASTIC_PORT (there is NO single ES-URL env). + # ⚠️ MUST be the canonical `scheme://host:port` form. A scheme-less value + # silently defaults to scheme=http; a port-less value silently defaults to + # port=9200 — either can break the ES connect against a TLS/non-9200 cluster. + # If your URL has a path/userinfo, set explicit scheme/host/port instead. + url: "https://es.example.com:9200" + # scheme: https # optional explicit override (wins over url) -> ELASTIC_SCHEME + # host: es.example.com # optional explicit override (wins over url) -> ELASTIC_HOST + # port: 9200 # optional explicit override (wins over url) -> ELASTIC_PORT + # K8s Secret you create (chart does NOT create it). Keys: es-auth-method, + # es-username, es-password, es-api-key, es-bearer-token. Leave empty for noauth ES. + credentialsSecretName: "sc4es-es-credentials" + default: true # the only cluster → bare-table routing + # Sidecar incoming auth OFF (intra-cluster trust). Federation reaches it plaintext + # over the in-cluster Service. For a Secret-backed sidecar auth, see 16.3 docs. + auth: + method: none diff --git a/softclient4es-federation/examples/three-region/README.md b/softclient4es-federation/examples/three-region/README.md new file mode 100644 index 0000000..caaf28c --- /dev/null +++ b/softclient4es-federation/examples/three-region/README.md @@ -0,0 +1,107 @@ +# Three-region federation (mixed ES versions) — Pro / Enterprise + +Federation + 3 region sidecars: **us-east-1 (ES 8)**, **eu-west-1 (ES 8)**, +**ap-south-1 (ES 9)**. One Flight SQL endpoint federating three Elasticsearch +clusters across regions and ES major versions. + +## ⚠️ Licensing — a Pro (or Enterprise) license is REQUIRED + +3 sidecars = 3 clusters. The federation enforces a per-platform cluster **quota** +at startup (it is the `maxClusters` quota, **not** a feature flag): + +| Clusters (sidecars) | Community (no license) | Pro | Enterprise | +|---|---|---|---| +| 1 | ✅ Ready (maxClusters=1) | ✅ | ✅ | +| 2–5 | ❌ federation CrashLoops (sys.exit) | ✅ (maxClusters=5) | ✅ | +| 6+ | ❌ | ❌ | ✅ (unlimited) | + +So this example **will not boot** without a Pro/Enterprise license. Create the +license Secret BEFORE `helm install`: + +```bash +kubectl create secret generic sc4es-pro-license --from-literal=license-key="$SC4ES_PRO_JWT" +``` + +(The federation surfaces the upgrade URL it emits at startup if you forget.) + +> ⚠️ **A Pro JWT alone is not enough — the federation IMAGE must be able to verify it.** +> Offline JWT verification is performed by a closed-source license manager that is +> bundled ONLY in the Pro/Enterprise-capable federation image. The Community/OSS image +> ships only the Community license manager, so it ignores any injected JWT, resolves to +> Community (`maxClusters=1`), and **still CrashLoops** on this 3-cluster example. The +> image must ALSO be able to resolve the license SIGNING public key offline (env +> `SOFTCLIENT4ES_LICENSE_PUBLIC_KEY`, or the issuer's JWKS endpoint must be reachable). +> Use a Pro/Enterprise-entitled federation image + provision the public key; consult the +> licensing/operator guide for the exact image + key provisioning. This precondition is +> tracked for the CI/install path (Story 16.5 FACT F / Story 16.1 OQ-5). A bare +> `--set image.tag` of the OSS snapshot image WILL CrashLoop even with a valid JWT Secret. + +## When to use this — three wedges + +- **GDPR data residency.** `eu-west-1` keeps EU data in the EU; the federation can + still run a cross-cluster JOIN that *references* it without copying rows across + borders (the JOIN executes in the federation; each region's data stays home). +- **SRE incident triage** (R1 marketing wedge). During an incident, one Flight SQL + endpoint lets an SRE `JOIN` logs/metrics/traces indices that live in **different + regional clusters** — no per-cluster console hopping, no ETL. A single query + correlates `us-east-1.errors` with `ap-south-1.upstream_latency`. +- **Mixed-version migration window.** Run ES 8 in `us-east-1` + `eu-west-1` while + `ap-south-1` is migrated to ES 9 — the federation routes across versions + transparently (the per-region sidecar image is chosen from `elasticsearchVersion`). + No "big bang" cutover; migrate one region at a time and keep federating throughout. + +## Topology + +``` + ┌──────────────────────────────────┐ + Flight SQL client │ Federation (Flight SQL :32020) │ + (BI / ADBC / JDBC) ──┼─► softclient4es-federation │ + │ health gRPC :32021 (aggregate) │ + └───┬────────────┬────────────┬─────┘ + │ │ │ in-cluster gRPC + ┌─────────────▼──┐ ┌──────▼───────┐ ┌─▼──────────────┐ + │ us-east-1 │ │ eu-west-1 │ │ ap-south-1 │ + │ sidecar (ES 8) │ │ sidecar(ES 8)│ │ sidecar (ES 9) │ + │ :32010 default │ │ :32010 │ │ :32010 │ + └────────┬───────┘ └──────┬───────┘ └───────┬────────┘ + ▼ ▼ ▼ + ┌────────────────┐ ┌────────────────┐ ┌────────────────┐ + │ ES 8 us-east-1│ │ ES 8 eu-west-1│ │ ES 9 ap-south-1│ + │ (US data) │ │ (EU data/GDPR) │ │ (migrating→ES9)│ + └────────────────┘ └────────────────┘ └────────────────┘ +``` + +## Install + +```bash +# 1. Pro license Secret (REQUIRED): +kubectl create secret generic sc4es-pro-license --from-literal=license-key="$SC4ES_PRO_JWT" + +# 2. Per-region ES + sidecar-auth Secrets (chart does NOT create these): +for r in us-east-1 eu-west-1 ap-south-1; do + kubectl create secret generic sc4es-es-$r \ + --from-literal=es-auth-method=basic --from-literal=es-username=elastic --from-literal=es-password='' + kubectl create secret generic sc4es-arrow-$r --from-literal=arrow-bearer-token='' +done + +# 3. Install: +helm install softclient4es-federation softclient4es-federation \ + -f examples/three-region/values.yaml +``` + +## Verify + +```bash +helm test softclient4es-federation # SHOW CATALOGS returns 3 +``` + +## Readiness & the all-or-nothing aggregate (important for multi-region) + +With `federation.probes.useGrpc: true` (default, K8s ≥ 1.27) the federation's gRPC +readiness aggregate is **all-or-nothing**: if **ANY ONE** region's sidecar is +unreachable, the federation Pod goes **NotReady** and is pulled from its Service — +so **every** query fails, including ones targeting the healthy regions. This is +"fail-closed" routing. For multi-region production where partial availability is +preferable (keep serving the healthy regions, fail only the queries that touch the +down region), set `federation.probes.useGrpc: false` (TCP readiness — the federation +stays Ready and degrades per-query). On K8s < 1.27 you MUST use `useGrpc: false`. diff --git a/softclient4es-federation/examples/three-region/values.yaml b/softclient4es-federation/examples/three-region/values.yaml new file mode 100644 index 0000000..aa61d8d --- /dev/null +++ b/softclient4es-federation/examples/three-region/values.yaml @@ -0,0 +1,63 @@ +# ───────────────────────────────────────────────────────────────────────────── +# SoftClient4ES Federation — THREE-REGION (mixed ES versions) example. +# +# Federation + 3 sidecars across us-east-1 (ES 8), eu-west-1 (ES 8), ap-south-1 (ES 9). +# Demonstrates GDPR data residency, cross-cluster JOIN, SRE incident triage, and a +# mixed-ES-version migration window (two regions on ES 8 while ap-south-1 is on ES 9). +# +# ⚠️ REQUIRES A PRO (or Enterprise) LICENSE. +# 3 sidecars = 3 clusters; Community caps at maxClusters=1, so WITHOUT a Pro/ +# Enterprise license the federation Pod CrashLoops by design (startup quota guard). +# Create a Secret `sc4es-pro-license` (data key `license-key` = your JWT) first. +# +# Install: +# helm install softclient4es-federation softclient4es-federation \ +# -f examples/three-region/values.yaml +# ───────────────────────────────────────────────────────────────────────────── + +# Pro/Enterprise license — REQUIRED for >=2 sidecars (maxClusters: Community=1, Pro=5, Ent=∞). +license: + secretName: "sc4es-pro-license" # data key `license-key` = JWT (override via licenseKeyKey) + +telemetry: + enabled: true + +federation: + probes: + # All-or-nothing gRPC readiness aggregate (VERIFIED): ANY unreachable sidecar + # makes the federation NotReady → ALL queries fail (incl. healthy regions). + # For multi-region partial-availability, set useGrpc: false (TCP readiness) — + # see README "Readiness & the all-or-nothing aggregate". + useGrpc: true + +sidecars: + # ── us-east-1 (Elasticsearch 8) — default region for bare-table routing ── + - name: us-east-1 + elasticsearchVersion: 8 + elasticsearch: + url: "https://es-us-east-1.example.com:9200" + credentialsSecretName: "sc4es-es-us-east-1" + auth: + method: bearer # sidecar incoming auth (single source of truth) + credentialsSecretName: "sc4es-arrow-us-east-1" # arrow-bearer-token -> ARROW_AUTH_* + CONFIG_FORCE_* + default: true + + # ── eu-west-1 (Elasticsearch 8) — GDPR data-residency region ── + - name: eu-west-1 + elasticsearchVersion: 8 + elasticsearch: + url: "https://es-eu-west-1.example.com:9200" + credentialsSecretName: "sc4es-es-eu-west-1" + auth: + method: bearer + credentialsSecretName: "sc4es-arrow-eu-west-1" + + # ── ap-south-1 (Elasticsearch 9) — region mid-migration to ES 9 ── + - name: ap-south-1 + elasticsearchVersion: 9 + elasticsearch: + url: "https://es-ap-south-1.example.com:9200" + credentialsSecretName: "sc4es-es-ap-south-1" + auth: + method: bearer + credentialsSecretName: "sc4es-arrow-ap-south-1" diff --git a/softclient4es-federation/templates/_helpers.tpl b/softclient4es-federation/templates/_helpers.tpl new file mode 100644 index 0000000..cfd9567 --- /dev/null +++ b/softclient4es-federation/templates/_helpers.tpl @@ -0,0 +1,176 @@ +{{/* Expand the name of the chart. */}} +{{- define "softclient4es-federation.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* Fully qualified app name (release-scoped, RFC1123, <=63 chars). */}} +{{- define "softclient4es-federation.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "softclient4es-federation.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* Common labels. */}} +{{- define "softclient4es-federation.labels" -}} +helm.sh/chart: {{ include "softclient4es-federation.chart" . }} +{{ include "softclient4es-federation.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: federation +app.kubernetes.io/part-of: softclient4es +{{- end -}} + +{{/* Selector labels — stable across upgrades; do NOT add version here. */}} +{{- define "softclient4es-federation.selectorLabels" -}} +app.kubernetes.io/name: {{ include "softclient4es-federation.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "softclient4es-federation.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} +{{- default (include "softclient4es-federation.fullname" .) .Values.serviceAccount.name -}} +{{- else -}} +{{- default "default" .Values.serviceAccount.name -}} +{{- end -}} +{{- end -}} + +{{/* +Per-ES-version sidecar image selection (Story 16.2). +Input: a dict {ctx: $, sidecar: $s}. Resolves $s.image.repository/.tag overrides, +else maps elasticsearchVersion (6|7|8|9) to the published image. The tag defaults to +.Chart.AppVersion, overridden by $s.image.tag. +*/}} +{{- define "softclient4es-federation.sidecarImage" -}} +{{- $s := .sidecar -}} +{{- $ctx := .ctx -}} +{{- $tag := $ctx.Chart.AppVersion -}} +{{- if $s.image }}{{- if $s.image.tag }}{{- $tag = $s.image.tag }}{{- end }}{{- end -}} +{{- if and $s.image $s.image.repository -}} +{{- printf "%s:%s" $s.image.repository $tag -}} +{{- else -}} +{{- $v := toString (required (printf "sidecar %q: elasticsearchVersion is required" $s.name) $s.elasticsearchVersion) -}} +{{- $repo := "" -}} +{{- if eq $v "6" -}}{{- $repo = "docker.io/softnetwork/softclient4es6-arrow-flight-sql" -}} +{{- else if eq $v "7" -}}{{- $repo = "docker.io/softnetwork/softclient4es7-arrow-flight-sql" -}} +{{- else if eq $v "8" -}}{{- $repo = "docker.io/softnetwork/softclient4es8-arrow-flight-sql" -}} +{{- else if eq $v "9" -}}{{- $repo = "docker.io/softnetwork/softclient4es9-arrow-flight-sql" -}} +{{- else -}}{{- fail (printf "sidecar %q: unsupported elasticsearchVersion %q (allowed: 6,7,8,9)" $s.name $v) -}} +{{- end -}} +{{- printf "%s:%s" $repo $tag -}} +{{- end -}} +{{- end -}} + +{{/* Per-sidecar fullname: -, RFC1123, <=63 chars. */}} +{{- define "softclient4es-federation.sidecarFullname" -}} +{{- $base := include "softclient4es-federation.fullname" .ctx -}} +{{- printf "%s-%s" $base .sidecar.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* Sidecar selector labels — stable across upgrades; identify the specific sidecar. */}} +{{- define "softclient4es-federation.sidecarSelectorLabels" -}} +app.kubernetes.io/name: {{ include "softclient4es-federation.name" .ctx }} +app.kubernetes.io/instance: {{ .ctx.Release.Name }} +app.kubernetes.io/component: sidecar +softclient4es.app/sidecar: {{ .sidecar.name }} +{{- end -}} + +{{/* Sidecar common labels. */}} +{{- define "softclient4es-federation.sidecarLabels" -}} +helm.sh/chart: {{ include "softclient4es-federation.chart" .ctx }} +{{ include "softclient4es-federation.sidecarSelectorLabels" . }} +{{- if .ctx.Chart.AppVersion }} +app.kubernetes.io/version: {{ .ctx.Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .ctx.Release.Service }} +app.kubernetes.io/part-of: softclient4es +{{- end -}} + +{{/* ES scheme: explicit es.scheme wins; else parsed from url (https://...); else http. */}} +{{- define "softclient4es-federation.esScheme" -}} +{{- $es := .es -}} +{{- if $es.scheme -}}{{ $es.scheme }} +{{- else if and $es.url (hasPrefix "https://" $es.url) -}}https +{{- else -}}http{{- end -}} +{{- end -}} + +{{/* ES host: explicit es.host wins; else the host portion of url; else localhost. */}} +{{- define "softclient4es-federation.esHost" -}} +{{- $es := .es -}} +{{- if $es.host -}}{{ $es.host }} +{{- else if $es.url -}} +{{- $hp := $es.url | trimPrefix "https://" | trimPrefix "http://" | trimSuffix "/" -}} +{{- (splitList ":" $hp) | first -}} +{{- else -}}localhost{{- end -}} +{{- end -}} + +{{/* ES port: explicit es.port wins; else the port portion of url; else 9200. */}} +{{- define "softclient4es-federation.esPort" -}} +{{- $es := .es -}} +{{- if $es.port -}}{{ $es.port }} +{{- else if $es.url -}} +{{- $hp := $es.url | trimPrefix "https://" | trimPrefix "http://" | trimSuffix "/" -}} +{{- $parts := splitList ":" $hp -}} +{{- if gt (len $parts) 1 -}}{{ last $parts }}{{- else -}}9200{{- end -}} +{{- else -}}9200{{- end -}} +{{- end -}} + +{{/* +Build the Typesafe Config `override_with_env_vars` env-var name for a federation->sidecar +credential leaf (Story 16.3). VERIFIED mangling (github.com/lightbend/config): strip +CONFIG_FORCE_, then `_`->`.`, `__`->`-`, `___`->`_`. So to TARGET a path containing dashes +we EMIT `__`. Input: dict {name: , key: }. The fixed path arrow.flight.federation.servers..credentials. -> + CONFIG_FORCE_arrow_flight_federation_servers___>_credentials___> +Example: name="prod-us" key="bearer-token" -> + CONFIG_FORCE_arrow_flight_federation_servers_prod__us_credentials_bearer__token +GUARD: the mangling is injective ONLY for RFC1123-label sidecar names (lowercase +alphanumeric + '-'). A name containing '_'/'.'/uppercase would mangle to a WRONG +CONFIG_FORCE_* path that silently does NOT override (-> federation validate() CrashLoop). +The sidecar-name validation block in federation-configmap.yaml enforces a strict RFC1123 +check, so by the time this helper runs the name is guaranteed dash-only — the +`replace "-" "__"` transform is then reversible/injective. +*/}} +{{- define "softclient4es-federation.configForceEnvName" -}} +{{- $name := .name | replace "-" "__" -}} +{{- $key := .key | replace "-" "__" -}} +{{- printf "CONFIG_FORCE_arrow_flight_federation_servers_%s_credentials_%s" $name $key -}} +{{- end -}} + +{{/* +ES Secret data-key for a given logical field, honoring sidecars[].elasticsearch.secretKeys +(Story 16.3). Input: dict {es: $s.elasticsearch, field: "username"} -> the Secret key +(default contract). Falls back to the FACT-C default when no override is given. +*/}} +{{- define "softclient4es-federation.esSecretKey" -}} +{{- $defaults := dict "authMethod" "es-auth-method" "username" "es-username" "password" "es-password" "apiKey" "es-api-key" "bearerToken" "es-bearer-token" -}} +{{- $field := .field -}} +{{- $override := "" -}} +{{- if .es.secretKeys -}}{{- $override = index .es.secretKeys $field | default "" -}}{{- end -}} +{{- if $override -}}{{ $override }}{{- else -}}{{ index $defaults $field }}{{- end -}} +{{- end -}} + +{{/* +Arrow-auth Secret data-key for a given logical field, honoring sidecars[].auth.secretKeys +(Story 16.3). Input: dict {auth: $s.auth, field: "bearerToken"} -> the Secret key +(default contract). Falls back to the FACT-C default when no override is given. +*/}} +{{- define "softclient4es-federation.arrowSecretKey" -}} +{{- $defaults := dict "username" "arrow-username" "password" "arrow-password" "bearerToken" "arrow-bearer-token" "apiKey" "arrow-api-key" -}} +{{- $field := .field -}} +{{- $override := "" -}} +{{- if .auth.secretKeys -}}{{- $override = index .auth.secretKeys $field | default "" -}}{{- end -}} +{{- if $override -}}{{ $override }}{{- else -}}{{ index $defaults $field }}{{- end -}} +{{- end -}} diff --git a/softclient4es-federation/templates/deployment.yaml b/softclient4es-federation/templates/deployment.yaml new file mode 100644 index 0000000..a8dfc33 --- /dev/null +++ b/softclient4es-federation/templates/deployment.yaml @@ -0,0 +1,216 @@ +{{- /* Story 16.3: does ANY sidecar use a Secret-backed (non-none) auth method? If so the + federation must (a) set -Dconfig.override_with_env_vars=true and (b) inject the + CONFIG_FORCE_* credential env from the SAME Secret the sidecar reads. */ -}} +{{- $secretAuth := false }} +{{- range $s := .Values.sidecars }}{{- if and $s.auth (ne (lower (default "none" $s.auth.method)) "none") $s.auth.credentialsSecretName }}{{- $secretAuth = true }}{{- end }}{{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "softclient4es-federation.fullname" . }} + labels: + {{- include "softclient4es-federation.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "softclient4es-federation.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "softclient4es-federation.selectorLabels" . | nindent 8 }} + {{- if gt (len .Values.sidecars) 0 }} + annotations: + # Roll the federation Pod whenever the rendered servers ConfigMap changes + # (add/remove/edit a sidecar) so `helm upgrade` actually takes effect. + checksum/config: {{ include (print $.Template.BasePath "/federation-configmap.yaml") . | sha256sum }} + {{- end }} + spec: + serviceAccountName: {{ include "softclient4es-federation.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: federation + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + ports: + - name: flight-sql + containerPort: {{ .Values.service.port }} + protocol: TCP + - name: health + containerPort: {{ .Values.federation.health.port }} + protocol: TCP + env: + - name: FEDERATION_HOST + value: "0.0.0.0" + - name: FEDERATION_PORT + value: {{ .Values.service.port | quote }} + - name: FEDERATION_MAX_MEMORY + value: {{ .Values.federation.maxMemory | quote }} + - name: FEDERATION_QUERY_TIMEOUT + value: {{ .Values.federation.queryTimeoutSeconds | quote }} + - name: FEDERATION_HEALTH_PORT + value: {{ .Values.federation.health.port | quote }} + - name: FEDERATION_HEALTH_PROBE_TIMEOUT + value: {{ .Values.federation.health.probeTimeoutSeconds | quote }} + - name: FEDERATION_DUCKDB_PATH + value: {{ .Values.federation.duckdb.path | quote }} + - name: FEDERATION_UPGRADE_URL + value: {{ .Values.federation.upgradeUrl | quote }} + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: {{ .Values.telemetry.enabled | quote }} + {{- if .Values.license.secretName }} + - name: SOFTCLIENT4ES_LICENSE_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.license.secretName }} + key: {{ .Values.license.licenseKeyKey | default "license-key" }} + optional: true + - name: SOFTCLIENT4ES_API_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.license.secretName }} + key: {{ .Values.license.apiKeyKey | default "api-key" }} + optional: true + {{- end }} + {{- /* Story 16.5 (FACT F #2): offline license public-key for air-gapped / + test-signed JWT verification (LicenseKeyVerifier.loadPublicKey Step 3). + Gated INDEPENDENTLY of license.secretName so the multi-sidecar CI can + point it at the same Secret. Renders nothing at the "" default, so the + golden and every example are byte-stable. */}} + {{- if .Values.license.publicKeySecretName }} + - name: SOFTCLIENT4ES_LICENSE_PUBLIC_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.license.publicKeySecretName }} + key: {{ .Values.license.publicKeyKey | default "license-public-key" }} + optional: true + {{- end }} + {{- if gt (len .Values.sidecars) 0 }} + # Point Typesafe Config at the mounted ConfigMap so ConfigFactory.load() + # picks up the `servers` map (which has NO env override). The native-packager + # bash launcher reads $JAVA_OPTS at runtime (VERIFIED) and prepends it to the + # JVM args; reference.conf still loads, so the FEDERATION_* env defaults stay. + # Story 16.3: when any sidecar uses Secret-backed auth, also enable + # override_with_env_vars so the CONFIG_FORCE_* env below force the credential + # leaf onto the loaded config BEFORE FederationConfig.validate() runs. + - name: JAVA_OPTS + value: "-Dconfig.file=/opt/docker/conf/application.conf{{ if and $secretAuth .Values.federation.credentialsFromEnv }} -Dconfig.override_with_env_vars=true{{ end }}" + {{- end }} + {{- /* Federation OUTGOING credential leaves the Secret as CONFIG_FORCE_* env (Story + 16.3, FACT A). ONE Secret feeds BOTH the sidecar (ARROW_AUTH_*) and the + federation here. The KEY/path is mangled (RFC1123-gated in the configmap); + the VALUE is a literal env string (no HOCON re-parse — special chars safe). */}} + {{- if .Values.federation.credentialsFromEnv }} + {{- range $s := .Values.sidecars }} + {{- with $s.auth }} + {{- if and (ne (lower (default "none" .method)) "none") .credentialsSecretName }} + {{- $m := lower .method }} + {{- if eq $m "basic" }} + - name: {{ include "softclient4es-federation.configForceEnvName" (dict "name" $s.name "key" "username") }} + valueFrom: + secretKeyRef: + name: {{ .credentialsSecretName }} + key: {{ include "softclient4es-federation.arrowSecretKey" (dict "auth" . "field" "username") }} + optional: true + - name: {{ include "softclient4es-federation.configForceEnvName" (dict "name" $s.name "key" "password") }} + valueFrom: + secretKeyRef: + name: {{ .credentialsSecretName }} + key: {{ include "softclient4es-federation.arrowSecretKey" (dict "auth" . "field" "password") }} + optional: true + {{- else if eq $m "bearer" }} + - name: {{ include "softclient4es-federation.configForceEnvName" (dict "name" $s.name "key" "bearer-token") }} + valueFrom: + secretKeyRef: + name: {{ .credentialsSecretName }} + key: {{ include "softclient4es-federation.arrowSecretKey" (dict "auth" . "field" "bearerToken") }} + optional: true + {{- else if or (eq $m "apikey") (eq $m "api") }} + - name: {{ include "softclient4es-federation.configForceEnvName" (dict "name" $s.name "key" "api-key") }} + valueFrom: + secretKeyRef: + name: {{ .credentialsSecretName }} + key: {{ include "softclient4es-federation.arrowSecretKey" (dict "auth" . "field" "apiKey") }} + optional: true + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + # --- Probes (FACT A / A1): TCP-socket only for the skeleton. --- + # The federation health endpoint is gRPC grpc.health.v1.Health and returns + # NOT_SERVING while servers={} (HealthService.scala:105-111), so a gRPC Check + # probe would fail. A TCP probe proves the process is listening, which is the + # correct liveness/readiness signal for a no-downstream skeleton. Story 16.2 + # switches readiness to a gRPC probe once sidecars exist. + livenessProbe: + tcpSocket: + port: health + initialDelaySeconds: {{ .Values.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.probes.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.federation.health.probeTimeoutSeconds }} + failureThreshold: {{ .Values.probes.liveness.failureThreshold }} + # Readiness flips to a native gRPC probe once sidecars make the SERVING + # aggregate meaningful (Story 16.2). ⚠️ ALL-OR-NOTHING: any one unreachable + # downstream → NotReady → ALL queries fail (fail-closed). Set + # federation.probes.useGrpc=false to keep TCP readiness (partial availability; + # required on K8s < 1.27 — native gRPC probes are GA only from 1.27). + {{- if and (gt (len .Values.sidecars) 0) .Values.federation.probes.useGrpc }} + readinessProbe: + grpc: + port: {{ .Values.federation.health.port }} + initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.probes.readiness.periodSeconds }} + failureThreshold: {{ .Values.probes.readiness.failureThreshold }} + {{- else }} + readinessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.probes.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.federation.health.probeTimeoutSeconds }} + failureThreshold: {{ .Values.probes.readiness.failureThreshold }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: scratch + mountPath: /tmp + {{- if gt (len .Values.sidecars) 0 }} + - name: federation-config + mountPath: /opt/docker/conf + readOnly: true + {{- end }} + volumes: + # readOnlyRootFilesystem=true requires a writable scratch dir: the org.duckdb + # JDBC driver extracts its native libduckdb_java library to java.io.tmpdir (/tmp) + # on every boot (PersistentDuckDB is instantiated unconditionally — even with + # path=":memory:"), and the federation stages CTAS / INSERT-with-JOIN Parquet + # under java.io.tmpdir (FederationConfig.scala:266-267 → /tmp/softclient4es/scratch). + # A file DuckDB path (federation.duckdb.path != ":memory:") would also live here. + - name: scratch + emptyDir: + sizeLimit: {{ .Values.scratch.sizeLimit }} + {{- if gt (len .Values.sidecars) 0 }} + - name: federation-config + configMap: + name: {{ include "softclient4es-federation.fullname" . }}-config + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/softclient4es-federation/templates/federation-configmap.yaml b/softclient4es-federation/templates/federation-configmap.yaml new file mode 100644 index 0000000..b50075c --- /dev/null +++ b/softclient4es-federation/templates/federation-configmap.yaml @@ -0,0 +1,102 @@ +{{- if gt (len .Values.sidecars) 0 }} +{{- /* Validate at most one default sidecar (matches FederationConfig fail-fast). */ -}} +{{- $defaults := list -}} +{{- range $s := .Values.sidecars }}{{- if $s.default }}{{- $defaults = append $defaults $s.name }}{{- end }}{{- end -}} +{{- if gt (len $defaults) 1 }}{{- fail (printf "At most one sidecar may be default=true; got: %v" $defaults) }}{{- end -}} +{{- /* Validate sidecar names are unique (each is a k8s resource suffix AND a servers. HOCON key). */ -}} +{{- $names := list -}} +{{- range $s := .Values.sidecars }}{{- $names = append $names (required "every sidecar requires a name" $s.name) }}{{- end -}} +{{- if ne (len $names) (len (uniq $names)) }}{{- fail (printf "sidecar names must be unique; got: %v" $names) }}{{- end -}} +{{- /* STRICT RFC1123-LABEL gate (Story 16.3): a sidecar name is BOTH a k8s resource suffix + AND the servers. HOCON key AND the source for the CONFIG_FORCE_* env-var path + (override_with_env_vars). The mangling `-`->`__` is injective ONLY for lowercase + alphanumeric + '-' names; a name with '_'/'.'/uppercase would mangle to a WRONG + CONFIG_FORCE_* path that silently fails to override -> federation validate() CrashLoop. + Fail at TEMPLATE time so the bug never reaches a cluster. */ -}} +{{- range $s := .Values.sidecars }} +{{- if not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $s.name) }}{{- fail (printf "sidecar %q: name must be a strict RFC1123 label (lowercase alphanumeric + '-', no leading/trailing '-'); it is used as a k8s resource suffix, a HOCON servers. key, and the CONFIG_FORCE_* override path (an '_'/'.'/uppercase would silently mis-target the federation credential override)" $s.name) }}{{- end }} +{{- end -}} +{{- /* FAIL-FAST on federation-outgoing credentials that cannot be delivered. + FlightCredentials.validate() (VERIFIED, FlightCredentials.scala:145-178) REJECTS a + credentials{} block whose method is basic/apikey/bearer but whose required fields are + empty -> FederationConfig.fromConfig throws -> federation sys.exit(1) (CrashLoop). + Story 16.3 (FACT A): a `credentialsSecretName` is now a VALID source — the value arrives + via CONFIG_FORCE_* (override_with_env_vars) on the federation Deployment, so the ConfigMap + renders method-only. We therefore fail ONLY when method != none AND there is NEITHER an + inline value NOR a Secret source; and additionally when a Secret IS set but + federation.credentialsFromEnv=false (the federation then literally cannot read it). */ -}} +{{- range $s := .Values.sidecars }} +{{- with $s.auth }} +{{- $m := lower (default "none" .method) }} +{{- if ne $m "none" }} +{{- $hasSecret := .credentialsSecretName }} +{{- if eq $m "basic" }}{{- if and (not $hasSecret) (or (not .username) (not .password)) }}{{- fail (printf "sidecar %q: auth.method=basic needs auth.credentialsSecretName (recommended) OR inline auth.username+auth.password" $s.name) }}{{- end }}{{- end }} +{{- if or (eq $m "apikey") (eq $m "api") }}{{- if and (not $hasSecret) (not .apiKey) }}{{- fail (printf "sidecar %q: auth.method=apikey needs auth.credentialsSecretName OR inline auth.apiKey" $s.name) }}{{- end }}{{- end }} +{{- if eq $m "bearer" }}{{- if and (not $hasSecret) (not .bearerToken) }}{{- fail (printf "sidecar %q: auth.method=bearer needs auth.credentialsSecretName OR inline auth.bearerToken" $s.name) }}{{- end }}{{- end }} +{{- if and $hasSecret (not $.Values.federation.credentialsFromEnv) }}{{- fail (printf "sidecar %q: auth.credentialsSecretName is set but federation.credentialsFromEnv=false — the federation cannot read the Secret; set federation.credentialsFromEnv=true (CONFIG_FORCE_* injection) or supply inline creds" $s.name) }}{{- end }} +{{- /* The federation's CONFIG_FORCE_* path is ALWAYS per-key (secretKeyRef.key = the semantic + key e.g. arrow-bearer-token). auth.useEnvFrom shapes the SIDECAR's Secret with env-var-NAME + keys (ARROW_AUTH_*), which the federation cannot read for its per-key override -> the + CONFIG_FORCE_* env would resolve empty -> federation CrashLoop. Reject the incompatible + combo at template time (fail-fast over a silent boot failure). */ -}} +{{- if and $hasSecret .useEnvFrom $.Values.federation.credentialsFromEnv }}{{- fail (printf "sidecar %q: auth.useEnvFrom=true is incompatible with a Secret-backed federation->sidecar credential — the federation reads the Secret per-key (e.g. key %q) but useEnvFrom requires the Secret's keys to be env-var NAMES (ARROW_AUTH_*). Use per-key mode (auth.useEnvFrom=false) for a Secret-backed non-none auth method, OR supply the federation cred inline" $s.name (include "softclient4es-federation.arrowSecretKey" (dict "auth" . "field" "bearerToken"))) }}{{- end }} +{{- end }} +{{- end }} +{{- end -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "softclient4es-federation.fullname" . }}-config + labels: + {{- include "softclient4es-federation.labels" . | nindent 4 }} +data: + application.conf: | + # GENERATED by the softclient4es-federation Helm chart — do not edit in-cluster. + # Populates arrow.flight.federation.servers from values.yaml `sidecars[]`. + # The scalar FEDERATION_* settings are still supplied via env vars on the + # Deployment; this file ONLY carries the `servers` map (which has no env override). + arrow.flight.federation { + servers { + {{- range $s := .Values.sidecars }} + {{ $s.name }} { + host = "{{ include "softclient4es-federation.sidecarFullname" (dict "ctx" $ "sidecar" $s) }}.{{ $.Release.Namespace }}.svc.cluster.local" + port = 32010 + {{- if $s.alias }} + alias = "{{ $s.alias }}" + {{- end }} + {{- if $s.default }} + default = true + {{- end }} + {{- with $s.auth }} + {{- if and .method (ne (lower .method) "none") }} + # Federation OUTGOING auth to this sidecar — single source of truth with the + # sidecar's ARROW_AUTH_* env. For a Secret-backed credential (credentialsSecretName) + # the VALUE is absent here and arrives via CONFIG_FORCE_* (override_with_env_vars) on + # the federation Deployment (Story 16.3, FACT A) — this block then renders method-only, + # and the override forces the value before FederationConfig.validate() runs. Inline + # values (dev/test) are rendered when present. + credentials { + method = "{{ lower .method }}" + {{- if .username }} + username = "{{ .username }}" + {{- end }} + {{- if .password }} + password = "{{ .password }}" + {{- end }} + {{- if .bearerToken }} + bearer-token = "{{ .bearerToken }}" + {{- end }} + {{- if .apiKey }} + api-key = "{{ .apiKey }}" + {{- end }} + } + {{- end }} + {{- end }} + {{- if $s.tls }} + tls = true + {{- end }} + } + {{- end }} + } + } +{{- end }} diff --git a/softclient4es-federation/templates/ingress.yaml b/softclient4es-federation/templates/ingress.yaml new file mode 100644 index 0000000..a6d5b37 --- /dev/null +++ b/softclient4es-federation/templates/ingress.yaml @@ -0,0 +1,54 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "softclient4es-federation.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "softclient4es-federation.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + # Flight SQL is gRPC — the controller MUST proxy gRPC backends, e.g. nginx: + # nginx.ingress.kubernetes.io/backend-protocol: "GRPC" + # (NOT "GRPCS": the federation Pod is PLAINTEXT h2c upstream — the Ingress terminates + # client TLS and forwards cleartext to the pod). HTTP/2-to-client needs a TLS listener, + # so a plaintext Ingress will not proxy Flight SQL — TLS-at-the-edge is the supported path. + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- /* TLS: explicit ingress.tls wins; else auto from federation.tls when enabled. */}} + {{- if .Values.ingress.tls }} + tls: + {{- toYaml .Values.ingress.tls | nindent 4 }} + {{- else if and .Values.federation.tls.enabled .Values.federation.tls.secretName }} + tls: + - secretName: {{ .Values.federation.tls.secretName }} + hosts: + {{- range .Values.ingress.hosts }} + {{- if .host }} + - {{ .host | quote }} + {{- end }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + {{- if .host }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType | default "Prefix" }} + backend: + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/softclient4es-federation/templates/service.yaml b/softclient4es-federation/templates/service.yaml new file mode 100644 index 0000000..602237c --- /dev/null +++ b/softclient4es-federation/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "softclient4es-federation.fullname" . }} + labels: + {{- include "softclient4es-federation.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - name: flight-sql + port: {{ .Values.service.port }} + targetPort: flight-sql + protocol: TCP + selector: + {{- include "softclient4es-federation.selectorLabels" . | nindent 4 }} diff --git a/softclient4es-federation/templates/serviceaccount.yaml b/softclient4es-federation/templates/serviceaccount.yaml new file mode 100644 index 0000000..9439372 --- /dev/null +++ b/softclient4es-federation/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "softclient4es-federation.serviceAccountName" . }} + labels: + {{- include "softclient4es-federation.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end -}} diff --git a/softclient4es-federation/templates/sidecar-deployment.yaml b/softclient4es-federation/templates/sidecar-deployment.yaml new file mode 100644 index 0000000..e68c11d --- /dev/null +++ b/softclient4es-federation/templates/sidecar-deployment.yaml @@ -0,0 +1,208 @@ +{{- range $i, $s := .Values.sidecars }} +{{- $sd := $.Values.sidecarDefaults }} +{{- $res := $s.resources | default $sd.resources }} +{{- /* A sidecar MUST point at a backing ES cluster — fail with an actionable message rather than a confusing nil-pointer when the `elasticsearch` block is omitted. */ -}} +{{- $es := required (printf "sidecar %q: an `elasticsearch` block is required (set elasticsearch.url, or explicit elasticsearch.host/scheme/port)" $s.name) $s.elasticsearch }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "softclient4es-federation.sidecarFullname" (dict "ctx" $ "sidecar" $s) }} + labels: + {{- include "softclient4es-federation.sidecarLabels" (dict "ctx" $ "sidecar" $s) | nindent 4 }} +spec: + replicas: {{ $s.replicaCount | default 1 }} + selector: + matchLabels: + {{- include "softclient4es-federation.sidecarSelectorLabels" (dict "ctx" $ "sidecar" $s) | nindent 6 }} + template: + metadata: + labels: + {{- include "softclient4es-federation.sidecarSelectorLabels" (dict "ctx" $ "sidecar" $s) | nindent 8 }} + spec: + serviceAccountName: {{ include "softclient4es-federation.serviceAccountName" $ }} + {{- with $.Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml $sd.podSecurityContext | nindent 8 }} + containers: + - name: sidecar + image: "{{ include "softclient4es-federation.sidecarImage" (dict "ctx" $ "sidecar" $s) }}" + imagePullPolicy: {{ $sd.image.pullPolicy }} + securityContext: + {{- toYaml $sd.securityContext | nindent 12 }} + ports: + - name: flight-sql + containerPort: 32010 + protocol: TCP + env: + # ── Arrow Flight SQL server (arrow.flight.*) ── + - name: ARROW_HOST + value: "0.0.0.0" + - name: ARROW_PORT + value: "32010" + - name: ARROW_BATCH_SIZE + value: {{ ($s.arrow).batchSize | default 1000 | quote }} + - name: ARROW_QUERY_TIMEOUT_SECONDS + value: {{ ($s.arrow).queryTimeoutSeconds | default 120 | quote }} + - name: ARROW_JOIN_MAX_MEMORY + value: {{ ($s.arrow).maxMemory | default "256m" | quote }} + - name: ARROW_JOIN_UPGRADE_URL + value: {{ $.Values.federation.upgradeUrl | quote }} + # ── Elasticsearch connection (elastic.credentials.* — VERIFIED, NOT a single URL) ── + - name: ELASTIC_SCHEME + value: {{ include "softclient4es-federation.esScheme" (dict "es" $s.elasticsearch) | quote }} + - name: ELASTIC_HOST + value: {{ include "softclient4es-federation.esHost" (dict "es" $s.elasticsearch) | quote }} + - name: ELASTIC_PORT + value: {{ include "softclient4es-federation.esPort" (dict "es" $s.elasticsearch) | quote }} + {{- /* Per-key secretKeyRef (Story 16.3): contract keys overridable via + elasticsearch.secretKeys. Skipped when useEnvFrom (whole-Secret mode). */}} + {{- if and $s.elasticsearch.credentialsSecretName (not $s.elasticsearch.useEnvFrom) }} + - name: ELASTIC_AUTH_METHOD + valueFrom: + secretKeyRef: + name: {{ $s.elasticsearch.credentialsSecretName }} + key: {{ include "softclient4es-federation.esSecretKey" (dict "es" $s.elasticsearch "field" "authMethod") }} + optional: true + - name: ELASTIC_CREDENTIALS_USERNAME + valueFrom: + secretKeyRef: + name: {{ $s.elasticsearch.credentialsSecretName }} + key: {{ include "softclient4es-federation.esSecretKey" (dict "es" $s.elasticsearch "field" "username") }} + optional: true + - name: ELASTIC_CREDENTIALS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $s.elasticsearch.credentialsSecretName }} + key: {{ include "softclient4es-federation.esSecretKey" (dict "es" $s.elasticsearch "field" "password") }} + optional: true + - name: ELASTIC_CREDENTIALS_API_KEY + valueFrom: + secretKeyRef: + name: {{ $s.elasticsearch.credentialsSecretName }} + key: {{ include "softclient4es-federation.esSecretKey" (dict "es" $s.elasticsearch "field" "apiKey") }} + optional: true + - name: ELASTIC_CREDENTIALS_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: {{ $s.elasticsearch.credentialsSecretName }} + key: {{ include "softclient4es-federation.esSecretKey" (dict "es" $s.elasticsearch "field" "bearerToken") }} + optional: true + {{- end }} + # ── Sidecar SERVER-SIDE auth (incoming) — single source of truth with the + # federation's outgoing servers..credentials (rendered in the ConfigMap) ── + {{- with $s.auth }} + - name: ARROW_AUTH_METHOD + value: {{ .method | default "none" | quote }} + {{- /* Per-key secretKeyRef (Story 16.3): contract keys overridable via + auth.secretKeys. Skipped when useEnvFrom (whole-Secret mode). NOTE: these + render whenever credentialsSecretName is set, regardless of method — with + method=none the sidecar ignores them (benign; documented in README). */}} + {{- if and .credentialsSecretName (not .useEnvFrom) }} + - name: ARROW_AUTH_USERNAME + valueFrom: + secretKeyRef: + name: {{ .credentialsSecretName }} + key: {{ include "softclient4es-federation.arrowSecretKey" (dict "auth" . "field" "username") }} + optional: true + - name: ARROW_AUTH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .credentialsSecretName }} + key: {{ include "softclient4es-federation.arrowSecretKey" (dict "auth" . "field" "password") }} + optional: true + - name: ARROW_AUTH_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: {{ .credentialsSecretName }} + key: {{ include "softclient4es-federation.arrowSecretKey" (dict "auth" . "field" "bearerToken") }} + optional: true + - name: ARROW_AUTH_API_KEY + valueFrom: + secretKeyRef: + name: {{ .credentialsSecretName }} + key: {{ include "softclient4es-federation.arrowSecretKey" (dict "auth" . "field" "apiKey") }} + optional: true + {{- end }} + {{- end }} + # ── Telemetry opt-out (shared with federation) ── + # VERIFIED: SOFTCLIENT4ES_TELEMETRY_ENABLED is honored by the sidecar too — the env + # override lives in elasticsql licensing reference.conf:30 (softclient4es.telemetry.enabled), + # read by TelemetryConfig.load, on which the arrow sidecar depends transitively. NOTE it is + # NOT in arrowServerSettings.dockerEnvVars (build.sbt:243-255 bakes only ELASTIC_*/ARROW_*), + # so the chart MUST set it explicitly here for the opt-out to reach the sidecar. + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: {{ $.Values.telemetry.enabled | quote }} + {{- if $.Values.license.secretName }} + - name: SOFTCLIENT4ES_LICENSE_KEY + valueFrom: + secretKeyRef: + name: {{ $.Values.license.secretName }} + key: {{ $.Values.license.licenseKeyKey | default "license-key" }} + optional: true + - name: SOFTCLIENT4ES_API_KEY + valueFrom: + secretKeyRef: + name: {{ $.Values.license.secretName }} + key: {{ $.Values.license.apiKeyKey | default "api-key" }} + optional: true + {{- end }} + {{- /* Optional whole-Secret envFrom mode (Story 16.3): the Secret's data keys + MUST already be the literal env-var names (ELASTIC_ and ARROW_AUTH_ prefixed) + — there is no key remapping in this mode. */}} + {{- if or (and $s.elasticsearch.credentialsSecretName $s.elasticsearch.useEnvFrom) (and $s.auth $s.auth.credentialsSecretName $s.auth.useEnvFrom) }} + envFrom: + {{- if and $s.elasticsearch.credentialsSecretName $s.elasticsearch.useEnvFrom }} + - secretRef: + name: {{ $s.elasticsearch.credentialsSecretName }} + optional: true + {{- end }} + {{- if and $s.auth $s.auth.credentialsSecretName $s.auth.useEnvFrom }} + - secretRef: + name: {{ $s.auth.credentialsSecretName }} + optional: true + {{- end }} + {{- end }} + # TCP probes — the sidecar has NO gRPC health service (VERIFIED); a raw TCP + # connect to ARROW_PORT proves the Flight SQL server is listening. + livenessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: {{ $sd.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ $sd.probes.liveness.periodSeconds }} + failureThreshold: {{ $sd.probes.liveness.failureThreshold }} + readinessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: {{ $sd.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ $sd.probes.readiness.periodSeconds }} + failureThreshold: {{ $sd.probes.readiness.failureThreshold }} + resources: + {{- toYaml $res | nindent 12 }} + volumeMounts: + - name: scratch + mountPath: /tmp + volumes: + # Writable /tmp: the sidecar runs the same-cluster JOIN engine on DuckDB + # (DuckDBJoinExecutor via LocalJoinHandler — VERIFIED), whose org.duckdb JDBC + # driver extracts its native libduckdb_java to java.io.tmpdir on first JOIN. + # readOnlyRootFilesystem=true without this would crash the first JOIN query. + - name: scratch + emptyDir: + sizeLimit: {{ $sd.scratch.sizeLimit }} + {{- with $.Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/softclient4es-federation/templates/sidecar-service.yaml b/softclient4es-federation/templates/sidecar-service.yaml new file mode 100644 index 0000000..f606744 --- /dev/null +++ b/softclient4es-federation/templates/sidecar-service.yaml @@ -0,0 +1,18 @@ +{{- range $i, $s := .Values.sidecars }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "softclient4es-federation.sidecarFullname" (dict "ctx" $ "sidecar" $s) }} + labels: + {{- include "softclient4es-federation.sidecarLabels" (dict "ctx" $ "sidecar" $s) | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32010 + targetPort: flight-sql + protocol: TCP + selector: + {{- include "softclient4es-federation.sidecarSelectorLabels" (dict "ctx" $ "sidecar" $s) | nindent 4 }} +{{- end }} diff --git a/softclient4es-federation/templates/tests/smoke-test.yaml b/softclient4es-federation/templates/tests/smoke-test.yaml new file mode 100644 index 0000000..2157bd9 --- /dev/null +++ b/softclient4es-federation/templates/tests/smoke-test.yaml @@ -0,0 +1,55 @@ +{{- if gt (len .Values.sidecars) 0 }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "softclient4es-federation.fullname" . }}-test-catalogs + labels: + {{- include "softclient4es-federation.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: show-catalogs + # Default: a public python image; CI + air-gapped clusters MUST pin `test.image` + # to a PRE-BAKED image with adbc_driver_flightsql installed (no runtime pip). + image: {{ .Values.test.image | default "python:3.12-slim" | quote }} + command: ["sh", "-c"] + args: + - | + set -e + # If the image is NOT pre-baked, install ADBC at runtime (needs PyPI reachability). + python -c "import adbc_driver_flightsql" 2>/dev/null || \ + pip install --quiet adbc_driver_flightsql=={{ .Values.test.adbcVersion | default "1.6.0" }} \ + adbc_driver_manager=={{ .Values.test.adbcVersion | default "1.6.0" }} + python - <<'PY' + import sys, time + import adbc_driver_flightsql.dbapi as dbapi + host = "{{ include "softclient4es-federation.fullname" . }}" + uri = f"grpc://{host}:{{ .Values.service.port }}" + expected = {{ len .Values.sidecars }} + # The federation only reports its catalogs once it has DISCOVERED every downstream + # (gRPC readiness is all-or-nothing). Retry the connect + GetCatalogs for up to ~60s. + deadline = time.time() + 60 + last_err = None + while time.time() < deadline: + try: + conn = dbapi.connect(uri) + # GetCatalogs: one catalog_name row per registered sidecar alias (VERIFIED + # FederationFlightProducer.getStreamCatalogs — needs NO default downstream). + cats = [c["catalog_name"] + for c in conn.adbc_get_objects(depth="catalogs").read_all().to_pylist()] + cats = [c for c in cats if c] # drop empty/system catalog if present + conn.close() + if len(cats) == expected: + print(f"OK: {len(cats)} catalogs: {cats}") + sys.exit(0) + last_err = f"expected {expected} catalogs, got {len(cats)}: {cats}" + except Exception as e: # connect refused while federation still NotReady + last_err = f"connect/get_objects error: {e}" + time.sleep(3) + print(f"FAIL: {last_err}", file=sys.stderr) + sys.exit(1) + PY +{{- end }} diff --git a/softclient4es-federation/tests/golden/default.yaml b/softclient4es-federation/tests/golden/default.yaml new file mode 100644 index 0000000..20c660b --- /dev/null +++ b/softclient4es-federation/tests/golden/default.yaml @@ -0,0 +1,156 @@ +--- +# Source: softclient4es-federation/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +automountServiceAccountToken: true +--- +# Source: softclient4es-federation/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32020 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed +--- +# Source: softclient4es-federation/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: federation + image: "docker.io/softnetwork/softclient4es-federation:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32020 + protocol: TCP + - name: health + containerPort: 32021 + protocol: TCP + env: + - name: FEDERATION_HOST + value: "0.0.0.0" + - name: FEDERATION_PORT + value: "32020" + - name: FEDERATION_MAX_MEMORY + value: "512m" + - name: FEDERATION_QUERY_TIMEOUT + value: "30" + - name: FEDERATION_HEALTH_PORT + value: "32021" + - name: FEDERATION_HEALTH_PROBE_TIMEOUT + value: "5" + - name: FEDERATION_DUCKDB_PATH + value: ":memory:" + - name: FEDERATION_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + # --- Probes (FACT A / A1): TCP-socket only for the skeleton. --- + # The federation health endpoint is gRPC grpc.health.v1.Health and returns + # NOT_SERVING while servers={} (HealthService.scala:105-111), so a gRPC Check + # probe would fail. A TCP probe proves the process is listening, which is the + # correct liveness/readiness signal for a no-downstream skeleton. Story 16.2 + # switches readiness to a gRPC probe once sidecars exist. + livenessProbe: + tcpSocket: + port: health + initialDelaySeconds: 20 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + # Readiness flips to a native gRPC probe once sidecars make the SERVING + # aggregate meaningful (Story 16.2). ⚠️ ALL-OR-NOTHING: any one unreachable + # downstream → NotReady → ALL queries fail (fail-closed). Set + # federation.probes.useGrpc=false to keep TCP readiness (partial availability; + # required on K8s < 1.27 — native gRPC probes are GA only from 1.27). + readinessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + volumeMounts: + - name: scratch + mountPath: /tmp + volumes: + # readOnlyRootFilesystem=true requires a writable scratch dir: the org.duckdb + # JDBC driver extracts its native libduckdb_java library to java.io.tmpdir (/tmp) + # on every boot (PersistentDuckDB is instantiated unconditionally — even with + # path=":memory:"), and the federation stages CTAS / INSERT-with-JOIN Parquet + # under java.io.tmpdir (FederationConfig.scala:266-267 → /tmp/softclient4es/scratch). + # A file DuckDB path (federation.duckdb.path != ":memory:") would also live here. + - name: scratch + emptyDir: + sizeLimit: 2Gi diff --git a/softclient4es-federation/tests/golden/example-heterogeneous-ready.yaml b/softclient4es-federation/tests/golden/example-heterogeneous-ready.yaml new file mode 100644 index 0000000..32e34bf --- /dev/null +++ b/softclient4es-federation/tests/golden/example-heterogeneous-ready.yaml @@ -0,0 +1,968 @@ +--- +# Source: softclient4es-federation/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +automountServiceAccountToken: true +--- +# Source: softclient4es-federation/templates/federation-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: fed-softclient4es-federation-config + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +data: + application.conf: | + # GENERATED by the softclient4es-federation Helm chart — do not edit in-cluster. + # Populates arrow.flight.federation.servers from values.yaml `sidecars[]`. + # The scalar FEDERATION_* settings are still supplied via env vars on the + # Deployment; this file ONLY carries the `servers` map (which has no env override). + arrow.flight.federation { + servers { + us-east-1 { + host = "fed-softclient4es-federation-us-east-1.default.svc.cluster.local" + port = 32010 + default = true + # Federation OUTGOING auth to this sidecar — single source of truth with the + # sidecar's ARROW_AUTH_* env. For a Secret-backed credential (credentialsSecretName) + # the VALUE is absent here and arrives via CONFIG_FORCE_* (override_with_env_vars) on + # the federation Deployment (Story 16.3, FACT A) — this block then renders method-only, + # and the override forces the value before FederationConfig.validate() runs. Inline + # values (dev/test) are rendered when present. + credentials { + method = "bearer" + } + } + eu-west-1 { + host = "fed-softclient4es-federation-eu-west-1.default.svc.cluster.local" + port = 32010 + # Federation OUTGOING auth to this sidecar — single source of truth with the + # sidecar's ARROW_AUTH_* env. For a Secret-backed credential (credentialsSecretName) + # the VALUE is absent here and arrives via CONFIG_FORCE_* (override_with_env_vars) on + # the federation Deployment (Story 16.3, FACT A) — this block then renders method-only, + # and the override forces the value before FederationConfig.validate() runs. Inline + # values (dev/test) are rendered when present. + credentials { + method = "bearer" + } + } + ap-south-1 { + host = "fed-softclient4es-federation-ap-south-1.default.svc.cluster.local" + port = 32010 + # Federation OUTGOING auth to this sidecar — single source of truth with the + # sidecar's ARROW_AUTH_* env. For a Secret-backed credential (credentialsSecretName) + # the VALUE is absent here and arrives via CONFIG_FORCE_* (override_with_env_vars) on + # the federation Deployment (Story 16.3, FACT A) — this block then renders method-only, + # and the override forces the value before FederationConfig.validate() runs. Inline + # values (dev/test) are rendered when present. + credentials { + method = "bearer" + } + } + } + } +--- +# Source: softclient4es-federation/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32020 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed +--- +# Source: softclient4es-federation/templates/sidecar-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation-us-east-1 + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: us-east-1 + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32010 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: us-east-1 +--- +# Source: softclient4es-federation/templates/sidecar-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation-eu-west-1 + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: eu-west-1 + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32010 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: eu-west-1 +--- +# Source: softclient4es-federation/templates/sidecar-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation-ap-south-1 + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: ap-south-1 + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32010 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: ap-south-1 +--- +# Source: softclient4es-federation/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + annotations: + # Roll the federation Pod whenever the rendered servers ConfigMap changes + # (add/remove/edit a sidecar) so `helm upgrade` actually takes effect. + checksum/config: e8fc04a9ca01bd0f0ff08964beca0bee9a2da40b729f47d4aa0b57a3fb2bac95 + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: federation + image: "docker.io/softnetwork/softclient4es-federation:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32020 + protocol: TCP + - name: health + containerPort: 32021 + protocol: TCP + env: + - name: FEDERATION_HOST + value: "0.0.0.0" + - name: FEDERATION_PORT + value: "32020" + - name: FEDERATION_MAX_MEMORY + value: "512m" + - name: FEDERATION_QUERY_TIMEOUT + value: "30" + - name: FEDERATION_HEALTH_PORT + value: "32021" + - name: FEDERATION_HEALTH_PROBE_TIMEOUT + value: "5" + - name: FEDERATION_DUCKDB_PATH + value: ":memory:" + - name: FEDERATION_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + - name: SOFTCLIENT4ES_LICENSE_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: license-key + optional: true + - name: SOFTCLIENT4ES_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: api-key + optional: true + # Point Typesafe Config at the mounted ConfigMap so ConfigFactory.load() + # picks up the `servers` map (which has NO env override). The native-packager + # bash launcher reads $JAVA_OPTS at runtime (VERIFIED) and prepends it to the + # JVM args; reference.conf still loads, so the FEDERATION_* env defaults stay. + # Story 16.3: when any sidecar uses Secret-backed auth, also enable + # override_with_env_vars so the CONFIG_FORCE_* env below force the credential + # leaf onto the loaded config BEFORE FederationConfig.validate() runs. + - name: JAVA_OPTS + value: "-Dconfig.file=/opt/docker/conf/application.conf -Dconfig.override_with_env_vars=true" + - name: CONFIG_FORCE_arrow_flight_federation_servers_us__east__1_credentials_bearer__token + valueFrom: + secretKeyRef: + name: sc4es-arrow-us-east-1 + key: arrow-bearer-token + optional: true + - name: CONFIG_FORCE_arrow_flight_federation_servers_eu__west__1_credentials_bearer__token + valueFrom: + secretKeyRef: + name: sc4es-arrow-eu-west-1 + key: arrow-bearer-token + optional: true + - name: CONFIG_FORCE_arrow_flight_federation_servers_ap__south__1_credentials_bearer__token + valueFrom: + secretKeyRef: + name: sc4es-arrow-ap-south-1 + key: arrow-bearer-token + optional: true + # --- Probes (FACT A / A1): TCP-socket only for the skeleton. --- + # The federation health endpoint is gRPC grpc.health.v1.Health and returns + # NOT_SERVING while servers={} (HealthService.scala:105-111), so a gRPC Check + # probe would fail. A TCP probe proves the process is listening, which is the + # correct liveness/readiness signal for a no-downstream skeleton. Story 16.2 + # switches readiness to a gRPC probe once sidecars exist. + livenessProbe: + tcpSocket: + port: health + initialDelaySeconds: 20 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + # Readiness flips to a native gRPC probe once sidecars make the SERVING + # aggregate meaningful (Story 16.2). ⚠️ ALL-OR-NOTHING: any one unreachable + # downstream → NotReady → ALL queries fail (fail-closed). Set + # federation.probes.useGrpc=false to keep TCP readiness (partial availability; + # required on K8s < 1.27 — native gRPC probes are GA only from 1.27). + readinessProbe: + grpc: + port: 32021 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + volumeMounts: + - name: scratch + mountPath: /tmp + - name: federation-config + mountPath: /opt/docker/conf + readOnly: true + volumes: + # readOnlyRootFilesystem=true requires a writable scratch dir: the org.duckdb + # JDBC driver extracts its native libduckdb_java library to java.io.tmpdir (/tmp) + # on every boot (PersistentDuckDB is instantiated unconditionally — even with + # path=":memory:"), and the federation stages CTAS / INSERT-with-JOIN Parquet + # under java.io.tmpdir (FederationConfig.scala:266-267 → /tmp/softclient4es/scratch). + # A file DuckDB path (federation.duckdb.path != ":memory:") would also live here. + - name: scratch + emptyDir: + sizeLimit: 2Gi + - name: federation-config + configMap: + name: fed-softclient4es-federation-config +--- +# Source: softclient4es-federation/templates/sidecar-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation-us-east-1 + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: us-east-1 + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: us-east-1 + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: us-east-1 + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: sidecar + image: "docker.io/softnetwork/softclient4es8-arrow-flight-sql:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32010 + protocol: TCP + env: + # ── Arrow Flight SQL server (arrow.flight.*) ── + - name: ARROW_HOST + value: "0.0.0.0" + - name: ARROW_PORT + value: "32010" + - name: ARROW_BATCH_SIZE + value: "1000" + - name: ARROW_QUERY_TIMEOUT_SECONDS + value: "120" + - name: ARROW_JOIN_MAX_MEMORY + value: "256m" + - name: ARROW_JOIN_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + # ── Elasticsearch connection (elastic.credentials.* — VERIFIED, NOT a single URL) ── + - name: ELASTIC_SCHEME + value: "https" + - name: ELASTIC_HOST + value: "es-us-east-1.example.com" + - name: ELASTIC_PORT + value: "9200" + - name: ELASTIC_AUTH_METHOD + valueFrom: + secretKeyRef: + name: sc4es-es-us-east-1 + key: es-auth-method + optional: true + - name: ELASTIC_CREDENTIALS_USERNAME + valueFrom: + secretKeyRef: + name: sc4es-es-us-east-1 + key: es-username + optional: true + - name: ELASTIC_CREDENTIALS_PASSWORD + valueFrom: + secretKeyRef: + name: sc4es-es-us-east-1 + key: es-password + optional: true + - name: ELASTIC_CREDENTIALS_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-es-us-east-1 + key: es-api-key + optional: true + - name: ELASTIC_CREDENTIALS_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: sc4es-es-us-east-1 + key: es-bearer-token + optional: true + # ── Sidecar SERVER-SIDE auth (incoming) — single source of truth with the + # federation's outgoing servers..credentials (rendered in the ConfigMap) ── + - name: ARROW_AUTH_METHOD + value: "bearer" + - name: ARROW_AUTH_USERNAME + valueFrom: + secretKeyRef: + name: sc4es-arrow-us-east-1 + key: arrow-username + optional: true + - name: ARROW_AUTH_PASSWORD + valueFrom: + secretKeyRef: + name: sc4es-arrow-us-east-1 + key: arrow-password + optional: true + - name: ARROW_AUTH_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: sc4es-arrow-us-east-1 + key: arrow-bearer-token + optional: true + - name: ARROW_AUTH_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-arrow-us-east-1 + key: arrow-api-key + optional: true + # ── Telemetry opt-out (shared with federation) ── + # VERIFIED: SOFTCLIENT4ES_TELEMETRY_ENABLED is honored by the sidecar too — the env + # override lives in elasticsql licensing reference.conf:30 (softclient4es.telemetry.enabled), + # read by TelemetryConfig.load, on which the arrow sidecar depends transitively. NOTE it is + # NOT in arrowServerSettings.dockerEnvVars (build.sbt:243-255 bakes only ELASTIC_*/ARROW_*), + # so the chart MUST set it explicitly here for the opt-out to reach the sidecar. + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + - name: SOFTCLIENT4ES_LICENSE_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: license-key + optional: true + - name: SOFTCLIENT4ES_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: api-key + optional: true + # TCP probes — the sidecar has NO gRPC health service (VERIFIED); a raw TCP + # connect to ARROW_PORT proves the Flight SQL server is listening. + livenessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 20 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 1536Mi + requests: + cpu: 500m + memory: 768Mi + volumeMounts: + - name: scratch + mountPath: /tmp + volumes: + # Writable /tmp: the sidecar runs the same-cluster JOIN engine on DuckDB + # (DuckDBJoinExecutor via LocalJoinHandler — VERIFIED), whose org.duckdb JDBC + # driver extracts its native libduckdb_java to java.io.tmpdir on first JOIN. + # readOnlyRootFilesystem=true without this would crash the first JOIN query. + - name: scratch + emptyDir: + sizeLimit: 1Gi +--- +# Source: softclient4es-federation/templates/sidecar-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation-eu-west-1 + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: eu-west-1 + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: eu-west-1 + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: eu-west-1 + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: sidecar + image: "docker.io/softnetwork/softclient4es8-arrow-flight-sql:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32010 + protocol: TCP + env: + # ── Arrow Flight SQL server (arrow.flight.*) ── + - name: ARROW_HOST + value: "0.0.0.0" + - name: ARROW_PORT + value: "32010" + - name: ARROW_BATCH_SIZE + value: "1000" + - name: ARROW_QUERY_TIMEOUT_SECONDS + value: "120" + - name: ARROW_JOIN_MAX_MEMORY + value: "256m" + - name: ARROW_JOIN_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + # ── Elasticsearch connection (elastic.credentials.* — VERIFIED, NOT a single URL) ── + - name: ELASTIC_SCHEME + value: "https" + - name: ELASTIC_HOST + value: "es-eu-west-1.example.com" + - name: ELASTIC_PORT + value: "9200" + - name: ELASTIC_AUTH_METHOD + valueFrom: + secretKeyRef: + name: sc4es-es-eu-west-1 + key: es-auth-method + optional: true + - name: ELASTIC_CREDENTIALS_USERNAME + valueFrom: + secretKeyRef: + name: sc4es-es-eu-west-1 + key: es-username + optional: true + - name: ELASTIC_CREDENTIALS_PASSWORD + valueFrom: + secretKeyRef: + name: sc4es-es-eu-west-1 + key: es-password + optional: true + - name: ELASTIC_CREDENTIALS_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-es-eu-west-1 + key: es-api-key + optional: true + - name: ELASTIC_CREDENTIALS_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: sc4es-es-eu-west-1 + key: es-bearer-token + optional: true + # ── Sidecar SERVER-SIDE auth (incoming) — single source of truth with the + # federation's outgoing servers..credentials (rendered in the ConfigMap) ── + - name: ARROW_AUTH_METHOD + value: "bearer" + - name: ARROW_AUTH_USERNAME + valueFrom: + secretKeyRef: + name: sc4es-arrow-eu-west-1 + key: arrow-username + optional: true + - name: ARROW_AUTH_PASSWORD + valueFrom: + secretKeyRef: + name: sc4es-arrow-eu-west-1 + key: arrow-password + optional: true + - name: ARROW_AUTH_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: sc4es-arrow-eu-west-1 + key: arrow-bearer-token + optional: true + - name: ARROW_AUTH_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-arrow-eu-west-1 + key: arrow-api-key + optional: true + # ── Telemetry opt-out (shared with federation) ── + # VERIFIED: SOFTCLIENT4ES_TELEMETRY_ENABLED is honored by the sidecar too — the env + # override lives in elasticsql licensing reference.conf:30 (softclient4es.telemetry.enabled), + # read by TelemetryConfig.load, on which the arrow sidecar depends transitively. NOTE it is + # NOT in arrowServerSettings.dockerEnvVars (build.sbt:243-255 bakes only ELASTIC_*/ARROW_*), + # so the chart MUST set it explicitly here for the opt-out to reach the sidecar. + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + - name: SOFTCLIENT4ES_LICENSE_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: license-key + optional: true + - name: SOFTCLIENT4ES_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: api-key + optional: true + # TCP probes — the sidecar has NO gRPC health service (VERIFIED); a raw TCP + # connect to ARROW_PORT proves the Flight SQL server is listening. + livenessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 20 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 1536Mi + requests: + cpu: 500m + memory: 768Mi + volumeMounts: + - name: scratch + mountPath: /tmp + volumes: + # Writable /tmp: the sidecar runs the same-cluster JOIN engine on DuckDB + # (DuckDBJoinExecutor via LocalJoinHandler — VERIFIED), whose org.duckdb JDBC + # driver extracts its native libduckdb_java to java.io.tmpdir on first JOIN. + # readOnlyRootFilesystem=true without this would crash the first JOIN query. + - name: scratch + emptyDir: + sizeLimit: 1Gi +--- +# Source: softclient4es-federation/templates/sidecar-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation-ap-south-1 + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: ap-south-1 + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: ap-south-1 + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: ap-south-1 + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: sidecar + image: "docker.io/softnetwork/softclient4es9-arrow-flight-sql:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32010 + protocol: TCP + env: + # ── Arrow Flight SQL server (arrow.flight.*) ── + - name: ARROW_HOST + value: "0.0.0.0" + - name: ARROW_PORT + value: "32010" + - name: ARROW_BATCH_SIZE + value: "1000" + - name: ARROW_QUERY_TIMEOUT_SECONDS + value: "120" + - name: ARROW_JOIN_MAX_MEMORY + value: "256m" + - name: ARROW_JOIN_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + # ── Elasticsearch connection (elastic.credentials.* — VERIFIED, NOT a single URL) ── + - name: ELASTIC_SCHEME + value: "https" + - name: ELASTIC_HOST + value: "es-ap-south-1.example.com" + - name: ELASTIC_PORT + value: "9200" + - name: ELASTIC_AUTH_METHOD + valueFrom: + secretKeyRef: + name: sc4es-es-ap-south-1 + key: es-auth-method + optional: true + - name: ELASTIC_CREDENTIALS_USERNAME + valueFrom: + secretKeyRef: + name: sc4es-es-ap-south-1 + key: es-username + optional: true + - name: ELASTIC_CREDENTIALS_PASSWORD + valueFrom: + secretKeyRef: + name: sc4es-es-ap-south-1 + key: es-password + optional: true + - name: ELASTIC_CREDENTIALS_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-es-ap-south-1 + key: es-api-key + optional: true + - name: ELASTIC_CREDENTIALS_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: sc4es-es-ap-south-1 + key: es-bearer-token + optional: true + # ── Sidecar SERVER-SIDE auth (incoming) — single source of truth with the + # federation's outgoing servers..credentials (rendered in the ConfigMap) ── + - name: ARROW_AUTH_METHOD + value: "bearer" + - name: ARROW_AUTH_USERNAME + valueFrom: + secretKeyRef: + name: sc4es-arrow-ap-south-1 + key: arrow-username + optional: true + - name: ARROW_AUTH_PASSWORD + valueFrom: + secretKeyRef: + name: sc4es-arrow-ap-south-1 + key: arrow-password + optional: true + - name: ARROW_AUTH_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: sc4es-arrow-ap-south-1 + key: arrow-bearer-token + optional: true + - name: ARROW_AUTH_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-arrow-ap-south-1 + key: arrow-api-key + optional: true + # ── Telemetry opt-out (shared with federation) ── + # VERIFIED: SOFTCLIENT4ES_TELEMETRY_ENABLED is honored by the sidecar too — the env + # override lives in elasticsql licensing reference.conf:30 (softclient4es.telemetry.enabled), + # read by TelemetryConfig.load, on which the arrow sidecar depends transitively. NOTE it is + # NOT in arrowServerSettings.dockerEnvVars (build.sbt:243-255 bakes only ELASTIC_*/ARROW_*), + # so the chart MUST set it explicitly here for the opt-out to reach the sidecar. + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + - name: SOFTCLIENT4ES_LICENSE_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: license-key + optional: true + - name: SOFTCLIENT4ES_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: api-key + optional: true + # TCP probes — the sidecar has NO gRPC health service (VERIFIED); a raw TCP + # connect to ARROW_PORT proves the Flight SQL server is listening. + livenessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 20 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 1536Mi + requests: + cpu: 500m + memory: 768Mi + volumeMounts: + - name: scratch + mountPath: /tmp + volumes: + # Writable /tmp: the sidecar runs the same-cluster JOIN engine on DuckDB + # (DuckDBJoinExecutor via LocalJoinHandler — VERIFIED), whose org.duckdb JDBC + # driver extracts its native libduckdb_java to java.io.tmpdir on first JOIN. + # readOnlyRootFilesystem=true without this would crash the first JOIN query. + - name: scratch + emptyDir: + sizeLimit: 1Gi +--- +# Source: softclient4es-federation/templates/tests/smoke-test.yaml +apiVersion: v1 +kind: Pod +metadata: + name: fed-softclient4es-federation-test-catalogs + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: show-catalogs + # Default: a public python image; CI + air-gapped clusters MUST pin `test.image` + # to a PRE-BAKED image with adbc_driver_flightsql installed (no runtime pip). + image: "python:3.12-slim" + command: ["sh", "-c"] + args: + - | + set -e + # If the image is NOT pre-baked, install ADBC at runtime (needs PyPI reachability). + python -c "import adbc_driver_flightsql" 2>/dev/null || \ + pip install --quiet adbc_driver_flightsql==1.6.0 \ + adbc_driver_manager==1.6.0 + python - <<'PY' + import sys, time + import adbc_driver_flightsql.dbapi as dbapi + host = "fed-softclient4es-federation" + uri = f"grpc://{host}:32020" + expected = 3 + # The federation only reports its catalogs once it has DISCOVERED every downstream + # (gRPC readiness is all-or-nothing). Retry the connect + GetCatalogs for up to ~60s. + deadline = time.time() + 60 + last_err = None + while time.time() < deadline: + try: + conn = dbapi.connect(uri) + # GetCatalogs: one catalog_name row per registered sidecar alias (VERIFIED + # FederationFlightProducer.getStreamCatalogs — needs NO default downstream). + cats = [c["catalog_name"] + for c in conn.adbc_get_objects(depth="catalogs").read_all().to_pylist()] + cats = [c for c in cats if c] # drop empty/system catalog if present + conn.close() + if len(cats) == expected: + print(f"OK: {len(cats)} catalogs: {cats}") + sys.exit(0) + last_err = f"expected {expected} catalogs, got {len(cats)}: {cats}" + except Exception as e: # connect refused while federation still NotReady + last_err = f"connect/get_objects error: {e}" + time.sleep(3) + print(f"FAIL: {last_err}", file=sys.stderr) + sys.exit(1) + PY diff --git a/softclient4es-federation/tests/golden/example-single-cluster.yaml b/softclient4es-federation/tests/golden/example-single-cluster.yaml new file mode 100644 index 0000000..a94e4b8 --- /dev/null +++ b/softclient4es-federation/tests/golden/example-single-cluster.yaml @@ -0,0 +1,441 @@ +--- +# Source: softclient4es-federation/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +automountServiceAccountToken: true +--- +# Source: softclient4es-federation/templates/federation-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: fed-softclient4es-federation-config + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +data: + application.conf: | + # GENERATED by the softclient4es-federation Helm chart — do not edit in-cluster. + # Populates arrow.flight.federation.servers from values.yaml `sidecars[]`. + # The scalar FEDERATION_* settings are still supplied via env vars on the + # Deployment; this file ONLY carries the `servers` map (which has no env override). + arrow.flight.federation { + servers { + primary { + host = "fed-softclient4es-federation-primary.default.svc.cluster.local" + port = 32010 + default = true + } + } + } +--- +# Source: softclient4es-federation/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32020 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed +--- +# Source: softclient4es-federation/templates/sidecar-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation-primary + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: primary + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32010 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: primary +--- +# Source: softclient4es-federation/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + annotations: + # Roll the federation Pod whenever the rendered servers ConfigMap changes + # (add/remove/edit a sidecar) so `helm upgrade` actually takes effect. + checksum/config: 1b75346d97567cae377a26f4e7f2e8d5c6920da45e8e7f9ca4e4b986169ece2b + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: federation + image: "docker.io/softnetwork/softclient4es-federation:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32020 + protocol: TCP + - name: health + containerPort: 32021 + protocol: TCP + env: + - name: FEDERATION_HOST + value: "0.0.0.0" + - name: FEDERATION_PORT + value: "32020" + - name: FEDERATION_MAX_MEMORY + value: "512m" + - name: FEDERATION_QUERY_TIMEOUT + value: "30" + - name: FEDERATION_HEALTH_PORT + value: "32021" + - name: FEDERATION_HEALTH_PROBE_TIMEOUT + value: "5" + - name: FEDERATION_DUCKDB_PATH + value: ":memory:" + - name: FEDERATION_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + # Point Typesafe Config at the mounted ConfigMap so ConfigFactory.load() + # picks up the `servers` map (which has NO env override). The native-packager + # bash launcher reads $JAVA_OPTS at runtime (VERIFIED) and prepends it to the + # JVM args; reference.conf still loads, so the FEDERATION_* env defaults stay. + # Story 16.3: when any sidecar uses Secret-backed auth, also enable + # override_with_env_vars so the CONFIG_FORCE_* env below force the credential + # leaf onto the loaded config BEFORE FederationConfig.validate() runs. + - name: JAVA_OPTS + value: "-Dconfig.file=/opt/docker/conf/application.conf" + # --- Probes (FACT A / A1): TCP-socket only for the skeleton. --- + # The federation health endpoint is gRPC grpc.health.v1.Health and returns + # NOT_SERVING while servers={} (HealthService.scala:105-111), so a gRPC Check + # probe would fail. A TCP probe proves the process is listening, which is the + # correct liveness/readiness signal for a no-downstream skeleton. Story 16.2 + # switches readiness to a gRPC probe once sidecars exist. + livenessProbe: + tcpSocket: + port: health + initialDelaySeconds: 20 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + # Readiness flips to a native gRPC probe once sidecars make the SERVING + # aggregate meaningful (Story 16.2). ⚠️ ALL-OR-NOTHING: any one unreachable + # downstream → NotReady → ALL queries fail (fail-closed). Set + # federation.probes.useGrpc=false to keep TCP readiness (partial availability; + # required on K8s < 1.27 — native gRPC probes are GA only from 1.27). + readinessProbe: + grpc: + port: 32021 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + volumeMounts: + - name: scratch + mountPath: /tmp + - name: federation-config + mountPath: /opt/docker/conf + readOnly: true + volumes: + # readOnlyRootFilesystem=true requires a writable scratch dir: the org.duckdb + # JDBC driver extracts its native libduckdb_java library to java.io.tmpdir (/tmp) + # on every boot (PersistentDuckDB is instantiated unconditionally — even with + # path=":memory:"), and the federation stages CTAS / INSERT-with-JOIN Parquet + # under java.io.tmpdir (FederationConfig.scala:266-267 → /tmp/softclient4es/scratch). + # A file DuckDB path (federation.duckdb.path != ":memory:") would also live here. + - name: scratch + emptyDir: + sizeLimit: 2Gi + - name: federation-config + configMap: + name: fed-softclient4es-federation-config +--- +# Source: softclient4es-federation/templates/sidecar-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation-primary + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: primary + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: primary + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: primary + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: sidecar + image: "docker.io/softnetwork/softclient4es8-arrow-flight-sql:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32010 + protocol: TCP + env: + # ── Arrow Flight SQL server (arrow.flight.*) ── + - name: ARROW_HOST + value: "0.0.0.0" + - name: ARROW_PORT + value: "32010" + - name: ARROW_BATCH_SIZE + value: "1000" + - name: ARROW_QUERY_TIMEOUT_SECONDS + value: "120" + - name: ARROW_JOIN_MAX_MEMORY + value: "256m" + - name: ARROW_JOIN_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + # ── Elasticsearch connection (elastic.credentials.* — VERIFIED, NOT a single URL) ── + - name: ELASTIC_SCHEME + value: "https" + - name: ELASTIC_HOST + value: "es.example.com" + - name: ELASTIC_PORT + value: "9200" + - name: ELASTIC_AUTH_METHOD + valueFrom: + secretKeyRef: + name: sc4es-es-credentials + key: es-auth-method + optional: true + - name: ELASTIC_CREDENTIALS_USERNAME + valueFrom: + secretKeyRef: + name: sc4es-es-credentials + key: es-username + optional: true + - name: ELASTIC_CREDENTIALS_PASSWORD + valueFrom: + secretKeyRef: + name: sc4es-es-credentials + key: es-password + optional: true + - name: ELASTIC_CREDENTIALS_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-es-credentials + key: es-api-key + optional: true + - name: ELASTIC_CREDENTIALS_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: sc4es-es-credentials + key: es-bearer-token + optional: true + # ── Sidecar SERVER-SIDE auth (incoming) — single source of truth with the + # federation's outgoing servers..credentials (rendered in the ConfigMap) ── + - name: ARROW_AUTH_METHOD + value: "none" + # ── Telemetry opt-out (shared with federation) ── + # VERIFIED: SOFTCLIENT4ES_TELEMETRY_ENABLED is honored by the sidecar too — the env + # override lives in elasticsql licensing reference.conf:30 (softclient4es.telemetry.enabled), + # read by TelemetryConfig.load, on which the arrow sidecar depends transitively. NOTE it is + # NOT in arrowServerSettings.dockerEnvVars (build.sbt:243-255 bakes only ELASTIC_*/ARROW_*), + # so the chart MUST set it explicitly here for the opt-out to reach the sidecar. + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + # TCP probes — the sidecar has NO gRPC health service (VERIFIED); a raw TCP + # connect to ARROW_PORT proves the Flight SQL server is listening. + livenessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 20 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 1536Mi + requests: + cpu: 500m + memory: 768Mi + volumeMounts: + - name: scratch + mountPath: /tmp + volumes: + # Writable /tmp: the sidecar runs the same-cluster JOIN engine on DuckDB + # (DuckDBJoinExecutor via LocalJoinHandler — VERIFIED), whose org.duckdb JDBC + # driver extracts its native libduckdb_java to java.io.tmpdir on first JOIN. + # readOnlyRootFilesystem=true without this would crash the first JOIN query. + - name: scratch + emptyDir: + sizeLimit: 1Gi +--- +# Source: softclient4es-federation/templates/tests/smoke-test.yaml +apiVersion: v1 +kind: Pod +metadata: + name: fed-softclient4es-federation-test-catalogs + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: show-catalogs + # Default: a public python image; CI + air-gapped clusters MUST pin `test.image` + # to a PRE-BAKED image with adbc_driver_flightsql installed (no runtime pip). + image: "python:3.12-slim" + command: ["sh", "-c"] + args: + - | + set -e + # If the image is NOT pre-baked, install ADBC at runtime (needs PyPI reachability). + python -c "import adbc_driver_flightsql" 2>/dev/null || \ + pip install --quiet adbc_driver_flightsql==1.6.0 \ + adbc_driver_manager==1.6.0 + python - <<'PY' + import sys, time + import adbc_driver_flightsql.dbapi as dbapi + host = "fed-softclient4es-federation" + uri = f"grpc://{host}:32020" + expected = 1 + # The federation only reports its catalogs once it has DISCOVERED every downstream + # (gRPC readiness is all-or-nothing). Retry the connect + GetCatalogs for up to ~60s. + deadline = time.time() + 60 + last_err = None + while time.time() < deadline: + try: + conn = dbapi.connect(uri) + # GetCatalogs: one catalog_name row per registered sidecar alias (VERIFIED + # FederationFlightProducer.getStreamCatalogs — needs NO default downstream). + cats = [c["catalog_name"] + for c in conn.adbc_get_objects(depth="catalogs").read_all().to_pylist()] + cats = [c for c in cats if c] # drop empty/system catalog if present + conn.close() + if len(cats) == expected: + print(f"OK: {len(cats)} catalogs: {cats}") + sys.exit(0) + last_err = f"expected {expected} catalogs, got {len(cats)}: {cats}" + except Exception as e: # connect refused while federation still NotReady + last_err = f"connect/get_objects error: {e}" + time.sleep(3) + print(f"FAIL: {last_err}", file=sys.stderr) + sys.exit(1) + PY diff --git a/softclient4es-federation/tests/golden/example-three-region.yaml b/softclient4es-federation/tests/golden/example-three-region.yaml new file mode 100644 index 0000000..32e34bf --- /dev/null +++ b/softclient4es-federation/tests/golden/example-three-region.yaml @@ -0,0 +1,968 @@ +--- +# Source: softclient4es-federation/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +automountServiceAccountToken: true +--- +# Source: softclient4es-federation/templates/federation-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: fed-softclient4es-federation-config + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +data: + application.conf: | + # GENERATED by the softclient4es-federation Helm chart — do not edit in-cluster. + # Populates arrow.flight.federation.servers from values.yaml `sidecars[]`. + # The scalar FEDERATION_* settings are still supplied via env vars on the + # Deployment; this file ONLY carries the `servers` map (which has no env override). + arrow.flight.federation { + servers { + us-east-1 { + host = "fed-softclient4es-federation-us-east-1.default.svc.cluster.local" + port = 32010 + default = true + # Federation OUTGOING auth to this sidecar — single source of truth with the + # sidecar's ARROW_AUTH_* env. For a Secret-backed credential (credentialsSecretName) + # the VALUE is absent here and arrives via CONFIG_FORCE_* (override_with_env_vars) on + # the federation Deployment (Story 16.3, FACT A) — this block then renders method-only, + # and the override forces the value before FederationConfig.validate() runs. Inline + # values (dev/test) are rendered when present. + credentials { + method = "bearer" + } + } + eu-west-1 { + host = "fed-softclient4es-federation-eu-west-1.default.svc.cluster.local" + port = 32010 + # Federation OUTGOING auth to this sidecar — single source of truth with the + # sidecar's ARROW_AUTH_* env. For a Secret-backed credential (credentialsSecretName) + # the VALUE is absent here and arrives via CONFIG_FORCE_* (override_with_env_vars) on + # the federation Deployment (Story 16.3, FACT A) — this block then renders method-only, + # and the override forces the value before FederationConfig.validate() runs. Inline + # values (dev/test) are rendered when present. + credentials { + method = "bearer" + } + } + ap-south-1 { + host = "fed-softclient4es-federation-ap-south-1.default.svc.cluster.local" + port = 32010 + # Federation OUTGOING auth to this sidecar — single source of truth with the + # sidecar's ARROW_AUTH_* env. For a Secret-backed credential (credentialsSecretName) + # the VALUE is absent here and arrives via CONFIG_FORCE_* (override_with_env_vars) on + # the federation Deployment (Story 16.3, FACT A) — this block then renders method-only, + # and the override forces the value before FederationConfig.validate() runs. Inline + # values (dev/test) are rendered when present. + credentials { + method = "bearer" + } + } + } + } +--- +# Source: softclient4es-federation/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32020 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed +--- +# Source: softclient4es-federation/templates/sidecar-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation-us-east-1 + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: us-east-1 + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32010 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: us-east-1 +--- +# Source: softclient4es-federation/templates/sidecar-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation-eu-west-1 + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: eu-west-1 + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32010 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: eu-west-1 +--- +# Source: softclient4es-federation/templates/sidecar-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation-ap-south-1 + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: ap-south-1 + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32010 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: ap-south-1 +--- +# Source: softclient4es-federation/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + annotations: + # Roll the federation Pod whenever the rendered servers ConfigMap changes + # (add/remove/edit a sidecar) so `helm upgrade` actually takes effect. + checksum/config: e8fc04a9ca01bd0f0ff08964beca0bee9a2da40b729f47d4aa0b57a3fb2bac95 + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: federation + image: "docker.io/softnetwork/softclient4es-federation:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32020 + protocol: TCP + - name: health + containerPort: 32021 + protocol: TCP + env: + - name: FEDERATION_HOST + value: "0.0.0.0" + - name: FEDERATION_PORT + value: "32020" + - name: FEDERATION_MAX_MEMORY + value: "512m" + - name: FEDERATION_QUERY_TIMEOUT + value: "30" + - name: FEDERATION_HEALTH_PORT + value: "32021" + - name: FEDERATION_HEALTH_PROBE_TIMEOUT + value: "5" + - name: FEDERATION_DUCKDB_PATH + value: ":memory:" + - name: FEDERATION_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + - name: SOFTCLIENT4ES_LICENSE_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: license-key + optional: true + - name: SOFTCLIENT4ES_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: api-key + optional: true + # Point Typesafe Config at the mounted ConfigMap so ConfigFactory.load() + # picks up the `servers` map (which has NO env override). The native-packager + # bash launcher reads $JAVA_OPTS at runtime (VERIFIED) and prepends it to the + # JVM args; reference.conf still loads, so the FEDERATION_* env defaults stay. + # Story 16.3: when any sidecar uses Secret-backed auth, also enable + # override_with_env_vars so the CONFIG_FORCE_* env below force the credential + # leaf onto the loaded config BEFORE FederationConfig.validate() runs. + - name: JAVA_OPTS + value: "-Dconfig.file=/opt/docker/conf/application.conf -Dconfig.override_with_env_vars=true" + - name: CONFIG_FORCE_arrow_flight_federation_servers_us__east__1_credentials_bearer__token + valueFrom: + secretKeyRef: + name: sc4es-arrow-us-east-1 + key: arrow-bearer-token + optional: true + - name: CONFIG_FORCE_arrow_flight_federation_servers_eu__west__1_credentials_bearer__token + valueFrom: + secretKeyRef: + name: sc4es-arrow-eu-west-1 + key: arrow-bearer-token + optional: true + - name: CONFIG_FORCE_arrow_flight_federation_servers_ap__south__1_credentials_bearer__token + valueFrom: + secretKeyRef: + name: sc4es-arrow-ap-south-1 + key: arrow-bearer-token + optional: true + # --- Probes (FACT A / A1): TCP-socket only for the skeleton. --- + # The federation health endpoint is gRPC grpc.health.v1.Health and returns + # NOT_SERVING while servers={} (HealthService.scala:105-111), so a gRPC Check + # probe would fail. A TCP probe proves the process is listening, which is the + # correct liveness/readiness signal for a no-downstream skeleton. Story 16.2 + # switches readiness to a gRPC probe once sidecars exist. + livenessProbe: + tcpSocket: + port: health + initialDelaySeconds: 20 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + # Readiness flips to a native gRPC probe once sidecars make the SERVING + # aggregate meaningful (Story 16.2). ⚠️ ALL-OR-NOTHING: any one unreachable + # downstream → NotReady → ALL queries fail (fail-closed). Set + # federation.probes.useGrpc=false to keep TCP readiness (partial availability; + # required on K8s < 1.27 — native gRPC probes are GA only from 1.27). + readinessProbe: + grpc: + port: 32021 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + volumeMounts: + - name: scratch + mountPath: /tmp + - name: federation-config + mountPath: /opt/docker/conf + readOnly: true + volumes: + # readOnlyRootFilesystem=true requires a writable scratch dir: the org.duckdb + # JDBC driver extracts its native libduckdb_java library to java.io.tmpdir (/tmp) + # on every boot (PersistentDuckDB is instantiated unconditionally — even with + # path=":memory:"), and the federation stages CTAS / INSERT-with-JOIN Parquet + # under java.io.tmpdir (FederationConfig.scala:266-267 → /tmp/softclient4es/scratch). + # A file DuckDB path (federation.duckdb.path != ":memory:") would also live here. + - name: scratch + emptyDir: + sizeLimit: 2Gi + - name: federation-config + configMap: + name: fed-softclient4es-federation-config +--- +# Source: softclient4es-federation/templates/sidecar-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation-us-east-1 + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: us-east-1 + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: us-east-1 + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: us-east-1 + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: sidecar + image: "docker.io/softnetwork/softclient4es8-arrow-flight-sql:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32010 + protocol: TCP + env: + # ── Arrow Flight SQL server (arrow.flight.*) ── + - name: ARROW_HOST + value: "0.0.0.0" + - name: ARROW_PORT + value: "32010" + - name: ARROW_BATCH_SIZE + value: "1000" + - name: ARROW_QUERY_TIMEOUT_SECONDS + value: "120" + - name: ARROW_JOIN_MAX_MEMORY + value: "256m" + - name: ARROW_JOIN_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + # ── Elasticsearch connection (elastic.credentials.* — VERIFIED, NOT a single URL) ── + - name: ELASTIC_SCHEME + value: "https" + - name: ELASTIC_HOST + value: "es-us-east-1.example.com" + - name: ELASTIC_PORT + value: "9200" + - name: ELASTIC_AUTH_METHOD + valueFrom: + secretKeyRef: + name: sc4es-es-us-east-1 + key: es-auth-method + optional: true + - name: ELASTIC_CREDENTIALS_USERNAME + valueFrom: + secretKeyRef: + name: sc4es-es-us-east-1 + key: es-username + optional: true + - name: ELASTIC_CREDENTIALS_PASSWORD + valueFrom: + secretKeyRef: + name: sc4es-es-us-east-1 + key: es-password + optional: true + - name: ELASTIC_CREDENTIALS_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-es-us-east-1 + key: es-api-key + optional: true + - name: ELASTIC_CREDENTIALS_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: sc4es-es-us-east-1 + key: es-bearer-token + optional: true + # ── Sidecar SERVER-SIDE auth (incoming) — single source of truth with the + # federation's outgoing servers..credentials (rendered in the ConfigMap) ── + - name: ARROW_AUTH_METHOD + value: "bearer" + - name: ARROW_AUTH_USERNAME + valueFrom: + secretKeyRef: + name: sc4es-arrow-us-east-1 + key: arrow-username + optional: true + - name: ARROW_AUTH_PASSWORD + valueFrom: + secretKeyRef: + name: sc4es-arrow-us-east-1 + key: arrow-password + optional: true + - name: ARROW_AUTH_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: sc4es-arrow-us-east-1 + key: arrow-bearer-token + optional: true + - name: ARROW_AUTH_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-arrow-us-east-1 + key: arrow-api-key + optional: true + # ── Telemetry opt-out (shared with federation) ── + # VERIFIED: SOFTCLIENT4ES_TELEMETRY_ENABLED is honored by the sidecar too — the env + # override lives in elasticsql licensing reference.conf:30 (softclient4es.telemetry.enabled), + # read by TelemetryConfig.load, on which the arrow sidecar depends transitively. NOTE it is + # NOT in arrowServerSettings.dockerEnvVars (build.sbt:243-255 bakes only ELASTIC_*/ARROW_*), + # so the chart MUST set it explicitly here for the opt-out to reach the sidecar. + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + - name: SOFTCLIENT4ES_LICENSE_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: license-key + optional: true + - name: SOFTCLIENT4ES_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: api-key + optional: true + # TCP probes — the sidecar has NO gRPC health service (VERIFIED); a raw TCP + # connect to ARROW_PORT proves the Flight SQL server is listening. + livenessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 20 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 1536Mi + requests: + cpu: 500m + memory: 768Mi + volumeMounts: + - name: scratch + mountPath: /tmp + volumes: + # Writable /tmp: the sidecar runs the same-cluster JOIN engine on DuckDB + # (DuckDBJoinExecutor via LocalJoinHandler — VERIFIED), whose org.duckdb JDBC + # driver extracts its native libduckdb_java to java.io.tmpdir on first JOIN. + # readOnlyRootFilesystem=true without this would crash the first JOIN query. + - name: scratch + emptyDir: + sizeLimit: 1Gi +--- +# Source: softclient4es-federation/templates/sidecar-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation-eu-west-1 + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: eu-west-1 + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: eu-west-1 + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: eu-west-1 + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: sidecar + image: "docker.io/softnetwork/softclient4es8-arrow-flight-sql:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32010 + protocol: TCP + env: + # ── Arrow Flight SQL server (arrow.flight.*) ── + - name: ARROW_HOST + value: "0.0.0.0" + - name: ARROW_PORT + value: "32010" + - name: ARROW_BATCH_SIZE + value: "1000" + - name: ARROW_QUERY_TIMEOUT_SECONDS + value: "120" + - name: ARROW_JOIN_MAX_MEMORY + value: "256m" + - name: ARROW_JOIN_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + # ── Elasticsearch connection (elastic.credentials.* — VERIFIED, NOT a single URL) ── + - name: ELASTIC_SCHEME + value: "https" + - name: ELASTIC_HOST + value: "es-eu-west-1.example.com" + - name: ELASTIC_PORT + value: "9200" + - name: ELASTIC_AUTH_METHOD + valueFrom: + secretKeyRef: + name: sc4es-es-eu-west-1 + key: es-auth-method + optional: true + - name: ELASTIC_CREDENTIALS_USERNAME + valueFrom: + secretKeyRef: + name: sc4es-es-eu-west-1 + key: es-username + optional: true + - name: ELASTIC_CREDENTIALS_PASSWORD + valueFrom: + secretKeyRef: + name: sc4es-es-eu-west-1 + key: es-password + optional: true + - name: ELASTIC_CREDENTIALS_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-es-eu-west-1 + key: es-api-key + optional: true + - name: ELASTIC_CREDENTIALS_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: sc4es-es-eu-west-1 + key: es-bearer-token + optional: true + # ── Sidecar SERVER-SIDE auth (incoming) — single source of truth with the + # federation's outgoing servers..credentials (rendered in the ConfigMap) ── + - name: ARROW_AUTH_METHOD + value: "bearer" + - name: ARROW_AUTH_USERNAME + valueFrom: + secretKeyRef: + name: sc4es-arrow-eu-west-1 + key: arrow-username + optional: true + - name: ARROW_AUTH_PASSWORD + valueFrom: + secretKeyRef: + name: sc4es-arrow-eu-west-1 + key: arrow-password + optional: true + - name: ARROW_AUTH_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: sc4es-arrow-eu-west-1 + key: arrow-bearer-token + optional: true + - name: ARROW_AUTH_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-arrow-eu-west-1 + key: arrow-api-key + optional: true + # ── Telemetry opt-out (shared with federation) ── + # VERIFIED: SOFTCLIENT4ES_TELEMETRY_ENABLED is honored by the sidecar too — the env + # override lives in elasticsql licensing reference.conf:30 (softclient4es.telemetry.enabled), + # read by TelemetryConfig.load, on which the arrow sidecar depends transitively. NOTE it is + # NOT in arrowServerSettings.dockerEnvVars (build.sbt:243-255 bakes only ELASTIC_*/ARROW_*), + # so the chart MUST set it explicitly here for the opt-out to reach the sidecar. + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + - name: SOFTCLIENT4ES_LICENSE_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: license-key + optional: true + - name: SOFTCLIENT4ES_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: api-key + optional: true + # TCP probes — the sidecar has NO gRPC health service (VERIFIED); a raw TCP + # connect to ARROW_PORT proves the Flight SQL server is listening. + livenessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 20 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 1536Mi + requests: + cpu: 500m + memory: 768Mi + volumeMounts: + - name: scratch + mountPath: /tmp + volumes: + # Writable /tmp: the sidecar runs the same-cluster JOIN engine on DuckDB + # (DuckDBJoinExecutor via LocalJoinHandler — VERIFIED), whose org.duckdb JDBC + # driver extracts its native libduckdb_java to java.io.tmpdir on first JOIN. + # readOnlyRootFilesystem=true without this would crash the first JOIN query. + - name: scratch + emptyDir: + sizeLimit: 1Gi +--- +# Source: softclient4es-federation/templates/sidecar-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation-ap-south-1 + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: ap-south-1 + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: ap-south-1 + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: ap-south-1 + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: sidecar + image: "docker.io/softnetwork/softclient4es9-arrow-flight-sql:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32010 + protocol: TCP + env: + # ── Arrow Flight SQL server (arrow.flight.*) ── + - name: ARROW_HOST + value: "0.0.0.0" + - name: ARROW_PORT + value: "32010" + - name: ARROW_BATCH_SIZE + value: "1000" + - name: ARROW_QUERY_TIMEOUT_SECONDS + value: "120" + - name: ARROW_JOIN_MAX_MEMORY + value: "256m" + - name: ARROW_JOIN_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + # ── Elasticsearch connection (elastic.credentials.* — VERIFIED, NOT a single URL) ── + - name: ELASTIC_SCHEME + value: "https" + - name: ELASTIC_HOST + value: "es-ap-south-1.example.com" + - name: ELASTIC_PORT + value: "9200" + - name: ELASTIC_AUTH_METHOD + valueFrom: + secretKeyRef: + name: sc4es-es-ap-south-1 + key: es-auth-method + optional: true + - name: ELASTIC_CREDENTIALS_USERNAME + valueFrom: + secretKeyRef: + name: sc4es-es-ap-south-1 + key: es-username + optional: true + - name: ELASTIC_CREDENTIALS_PASSWORD + valueFrom: + secretKeyRef: + name: sc4es-es-ap-south-1 + key: es-password + optional: true + - name: ELASTIC_CREDENTIALS_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-es-ap-south-1 + key: es-api-key + optional: true + - name: ELASTIC_CREDENTIALS_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: sc4es-es-ap-south-1 + key: es-bearer-token + optional: true + # ── Sidecar SERVER-SIDE auth (incoming) — single source of truth with the + # federation's outgoing servers..credentials (rendered in the ConfigMap) ── + - name: ARROW_AUTH_METHOD + value: "bearer" + - name: ARROW_AUTH_USERNAME + valueFrom: + secretKeyRef: + name: sc4es-arrow-ap-south-1 + key: arrow-username + optional: true + - name: ARROW_AUTH_PASSWORD + valueFrom: + secretKeyRef: + name: sc4es-arrow-ap-south-1 + key: arrow-password + optional: true + - name: ARROW_AUTH_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: sc4es-arrow-ap-south-1 + key: arrow-bearer-token + optional: true + - name: ARROW_AUTH_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-arrow-ap-south-1 + key: arrow-api-key + optional: true + # ── Telemetry opt-out (shared with federation) ── + # VERIFIED: SOFTCLIENT4ES_TELEMETRY_ENABLED is honored by the sidecar too — the env + # override lives in elasticsql licensing reference.conf:30 (softclient4es.telemetry.enabled), + # read by TelemetryConfig.load, on which the arrow sidecar depends transitively. NOTE it is + # NOT in arrowServerSettings.dockerEnvVars (build.sbt:243-255 bakes only ELASTIC_*/ARROW_*), + # so the chart MUST set it explicitly here for the opt-out to reach the sidecar. + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + - name: SOFTCLIENT4ES_LICENSE_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: license-key + optional: true + - name: SOFTCLIENT4ES_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-pro-license + key: api-key + optional: true + # TCP probes — the sidecar has NO gRPC health service (VERIFIED); a raw TCP + # connect to ARROW_PORT proves the Flight SQL server is listening. + livenessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 20 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 1536Mi + requests: + cpu: 500m + memory: 768Mi + volumeMounts: + - name: scratch + mountPath: /tmp + volumes: + # Writable /tmp: the sidecar runs the same-cluster JOIN engine on DuckDB + # (DuckDBJoinExecutor via LocalJoinHandler — VERIFIED), whose org.duckdb JDBC + # driver extracts its native libduckdb_java to java.io.tmpdir on first JOIN. + # readOnlyRootFilesystem=true without this would crash the first JOIN query. + - name: scratch + emptyDir: + sizeLimit: 1Gi +--- +# Source: softclient4es-federation/templates/tests/smoke-test.yaml +apiVersion: v1 +kind: Pod +metadata: + name: fed-softclient4es-federation-test-catalogs + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: show-catalogs + # Default: a public python image; CI + air-gapped clusters MUST pin `test.image` + # to a PRE-BAKED image with adbc_driver_flightsql installed (no runtime pip). + image: "python:3.12-slim" + command: ["sh", "-c"] + args: + - | + set -e + # If the image is NOT pre-baked, install ADBC at runtime (needs PyPI reachability). + python -c "import adbc_driver_flightsql" 2>/dev/null || \ + pip install --quiet adbc_driver_flightsql==1.6.0 \ + adbc_driver_manager==1.6.0 + python - <<'PY' + import sys, time + import adbc_driver_flightsql.dbapi as dbapi + host = "fed-softclient4es-federation" + uri = f"grpc://{host}:32020" + expected = 3 + # The federation only reports its catalogs once it has DISCOVERED every downstream + # (gRPC readiness is all-or-nothing). Retry the connect + GetCatalogs for up to ~60s. + deadline = time.time() + 60 + last_err = None + while time.time() < deadline: + try: + conn = dbapi.connect(uri) + # GetCatalogs: one catalog_name row per registered sidecar alias (VERIFIED + # FederationFlightProducer.getStreamCatalogs — needs NO default downstream). + cats = [c["catalog_name"] + for c in conn.adbc_get_objects(depth="catalogs").read_all().to_pylist()] + cats = [c for c in cats if c] # drop empty/system catalog if present + conn.close() + if len(cats) == expected: + print(f"OK: {len(cats)} catalogs: {cats}") + sys.exit(0) + last_err = f"expected {expected} catalogs, got {len(cats)}: {cats}" + except Exception as e: # connect refused while federation still NotReady + last_err = f"connect/get_objects error: {e}" + time.sleep(3) + print(f"FAIL: {last_err}", file=sys.stderr) + sys.exit(1) + PY diff --git a/softclient4es-federation/tests/golden/ingress-tls.yaml b/softclient4es-federation/tests/golden/ingress-tls.yaml new file mode 100644 index 0000000..c124858 --- /dev/null +++ b/softclient4es-federation/tests/golden/ingress-tls.yaml @@ -0,0 +1,448 @@ +--- +# Source: softclient4es-federation/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +automountServiceAccountToken: true +--- +# Source: softclient4es-federation/templates/federation-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: fed-softclient4es-federation-config + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +data: + application.conf: | + # GENERATED by the softclient4es-federation Helm chart — do not edit in-cluster. + # Populates arrow.flight.federation.servers from values.yaml `sidecars[]`. + # The scalar FEDERATION_* settings are still supplied via env vars on the + # Deployment; this file ONLY carries the `servers` map (which has no env override). + arrow.flight.federation { + servers { + prod-us { + host = "fed-softclient4es-federation-prod-us.default.svc.cluster.local" + port = 32010 + default = true + } + } + } +--- +# Source: softclient4es-federation/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32020 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed +--- +# Source: softclient4es-federation/templates/sidecar-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation-prod-us + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-us + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32010 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-us +--- +# Source: softclient4es-federation/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + annotations: + # Roll the federation Pod whenever the rendered servers ConfigMap changes + # (add/remove/edit a sidecar) so `helm upgrade` actually takes effect. + checksum/config: 09457749b0df680bf9a554b7867d5434503be84e4aa95d470b279614c251837e + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: federation + image: "docker.io/softnetwork/softclient4es-federation:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32020 + protocol: TCP + - name: health + containerPort: 32021 + protocol: TCP + env: + - name: FEDERATION_HOST + value: "0.0.0.0" + - name: FEDERATION_PORT + value: "32020" + - name: FEDERATION_MAX_MEMORY + value: "512m" + - name: FEDERATION_QUERY_TIMEOUT + value: "30" + - name: FEDERATION_HEALTH_PORT + value: "32021" + - name: FEDERATION_HEALTH_PROBE_TIMEOUT + value: "5" + - name: FEDERATION_DUCKDB_PATH + value: ":memory:" + - name: FEDERATION_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + # Point Typesafe Config at the mounted ConfigMap so ConfigFactory.load() + # picks up the `servers` map (which has NO env override). The native-packager + # bash launcher reads $JAVA_OPTS at runtime (VERIFIED) and prepends it to the + # JVM args; reference.conf still loads, so the FEDERATION_* env defaults stay. + # Story 16.3: when any sidecar uses Secret-backed auth, also enable + # override_with_env_vars so the CONFIG_FORCE_* env below force the credential + # leaf onto the loaded config BEFORE FederationConfig.validate() runs. + - name: JAVA_OPTS + value: "-Dconfig.file=/opt/docker/conf/application.conf" + # --- Probes (FACT A / A1): TCP-socket only for the skeleton. --- + # The federation health endpoint is gRPC grpc.health.v1.Health and returns + # NOT_SERVING while servers={} (HealthService.scala:105-111), so a gRPC Check + # probe would fail. A TCP probe proves the process is listening, which is the + # correct liveness/readiness signal for a no-downstream skeleton. Story 16.2 + # switches readiness to a gRPC probe once sidecars exist. + livenessProbe: + tcpSocket: + port: health + initialDelaySeconds: 20 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + # Readiness flips to a native gRPC probe once sidecars make the SERVING + # aggregate meaningful (Story 16.2). ⚠️ ALL-OR-NOTHING: any one unreachable + # downstream → NotReady → ALL queries fail (fail-closed). Set + # federation.probes.useGrpc=false to keep TCP readiness (partial availability; + # required on K8s < 1.27 — native gRPC probes are GA only from 1.27). + readinessProbe: + grpc: + port: 32021 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + volumeMounts: + - name: scratch + mountPath: /tmp + - name: federation-config + mountPath: /opt/docker/conf + readOnly: true + volumes: + # readOnlyRootFilesystem=true requires a writable scratch dir: the org.duckdb + # JDBC driver extracts its native libduckdb_java library to java.io.tmpdir (/tmp) + # on every boot (PersistentDuckDB is instantiated unconditionally — even with + # path=":memory:"), and the federation stages CTAS / INSERT-with-JOIN Parquet + # under java.io.tmpdir (FederationConfig.scala:266-267 → /tmp/softclient4es/scratch). + # A file DuckDB path (federation.duckdb.path != ":memory:") would also live here. + - name: scratch + emptyDir: + sizeLimit: 2Gi + - name: federation-config + configMap: + name: fed-softclient4es-federation-config +--- +# Source: softclient4es-federation/templates/sidecar-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation-prod-us + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-us + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-us + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-us + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: sidecar + image: "docker.io/softnetwork/softclient4es8-arrow-flight-sql:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32010 + protocol: TCP + env: + # ── Arrow Flight SQL server (arrow.flight.*) ── + - name: ARROW_HOST + value: "0.0.0.0" + - name: ARROW_PORT + value: "32010" + - name: ARROW_BATCH_SIZE + value: "1000" + - name: ARROW_QUERY_TIMEOUT_SECONDS + value: "120" + - name: ARROW_JOIN_MAX_MEMORY + value: "256m" + - name: ARROW_JOIN_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + # ── Elasticsearch connection (elastic.credentials.* — VERIFIED, NOT a single URL) ── + - name: ELASTIC_SCHEME + value: "https" + - name: ELASTIC_HOST + value: "es-us.example.com" + - name: ELASTIC_PORT + value: "9200" + # ── Sidecar SERVER-SIDE auth (incoming) — single source of truth with the + # federation's outgoing servers..credentials (rendered in the ConfigMap) ── + # ── Telemetry opt-out (shared with federation) ── + # VERIFIED: SOFTCLIENT4ES_TELEMETRY_ENABLED is honored by the sidecar too — the env + # override lives in elasticsql licensing reference.conf:30 (softclient4es.telemetry.enabled), + # read by TelemetryConfig.load, on which the arrow sidecar depends transitively. NOTE it is + # NOT in arrowServerSettings.dockerEnvVars (build.sbt:243-255 bakes only ELASTIC_*/ARROW_*), + # so the chart MUST set it explicitly here for the opt-out to reach the sidecar. + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + # TCP probes — the sidecar has NO gRPC health service (VERIFIED); a raw TCP + # connect to ARROW_PORT proves the Flight SQL server is listening. + livenessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 20 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 1536Mi + requests: + cpu: 500m + memory: 768Mi + volumeMounts: + - name: scratch + mountPath: /tmp + volumes: + # Writable /tmp: the sidecar runs the same-cluster JOIN engine on DuckDB + # (DuckDBJoinExecutor via LocalJoinHandler — VERIFIED), whose org.duckdb JDBC + # driver extracts its native libduckdb_java to java.io.tmpdir on first JOIN. + # readOnlyRootFilesystem=true without this would crash the first JOIN query. + - name: scratch + emptyDir: + sizeLimit: 1Gi +--- +# Source: softclient4es-federation/templates/ingress.yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es + annotations: + # Flight SQL is gRPC — the controller MUST proxy gRPC backends, e.g. nginx: + # nginx.ingress.kubernetes.io/backend-protocol: "GRPC" + # (NOT "GRPCS": the federation Pod is PLAINTEXT h2c upstream — the Ingress terminates + # client TLS and forwards cleartext to the pod). HTTP/2-to-client needs a TLS listener, + # so a plaintext Ingress will not proxy Flight SQL — TLS-at-the-edge is the supported path. + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/backend-protocol: GRPC +spec: + ingressClassName: nginx + tls: + - secretName: fed-tls + hosts: + - "fed.example.com" + rules: + - host: "fed.example.com" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: fed-softclient4es-federation + port: + number: 32020 +--- +# Source: softclient4es-federation/templates/tests/smoke-test.yaml +apiVersion: v1 +kind: Pod +metadata: + name: fed-softclient4es-federation-test-catalogs + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: show-catalogs + # Default: a public python image; CI + air-gapped clusters MUST pin `test.image` + # to a PRE-BAKED image with adbc_driver_flightsql installed (no runtime pip). + image: "python:3.12-slim" + command: ["sh", "-c"] + args: + - | + set -e + # If the image is NOT pre-baked, install ADBC at runtime (needs PyPI reachability). + python -c "import adbc_driver_flightsql" 2>/dev/null || \ + pip install --quiet adbc_driver_flightsql==1.6.0 \ + adbc_driver_manager==1.6.0 + python - <<'PY' + import sys, time + import adbc_driver_flightsql.dbapi as dbapi + host = "fed-softclient4es-federation" + uri = f"grpc://{host}:32020" + expected = 1 + # The federation only reports its catalogs once it has DISCOVERED every downstream + # (gRPC readiness is all-or-nothing). Retry the connect + GetCatalogs for up to ~60s. + deadline = time.time() + 60 + last_err = None + while time.time() < deadline: + try: + conn = dbapi.connect(uri) + # GetCatalogs: one catalog_name row per registered sidecar alias (VERIFIED + # FederationFlightProducer.getStreamCatalogs — needs NO default downstream). + cats = [c["catalog_name"] + for c in conn.adbc_get_objects(depth="catalogs").read_all().to_pylist()] + cats = [c for c in cats if c] # drop empty/system catalog if present + conn.close() + if len(cats) == expected: + print(f"OK: {len(cats)} catalogs: {cats}") + sys.exit(0) + last_err = f"expected {expected} catalogs, got {len(cats)}: {cats}" + except Exception as e: # connect refused while federation still NotReady + last_err = f"connect/get_objects error: {e}" + time.sleep(3) + print(f"FAIL: {last_err}", file=sys.stderr) + sys.exit(1) + PY diff --git a/softclient4es-federation/tests/golden/secret-auth.yaml b/softclient4es-federation/tests/golden/secret-auth.yaml new file mode 100644 index 0000000..bc47a30 --- /dev/null +++ b/softclient4es-federation/tests/golden/secret-auth.yaml @@ -0,0 +1,480 @@ +--- +# Source: softclient4es-federation/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +automountServiceAccountToken: true +--- +# Source: softclient4es-federation/templates/federation-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: fed-softclient4es-federation-config + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +data: + application.conf: | + # GENERATED by the softclient4es-federation Helm chart — do not edit in-cluster. + # Populates arrow.flight.federation.servers from values.yaml `sidecars[]`. + # The scalar FEDERATION_* settings are still supplied via env vars on the + # Deployment; this file ONLY carries the `servers` map (which has no env override). + arrow.flight.federation { + servers { + prod-us { + host = "fed-softclient4es-federation-prod-us.default.svc.cluster.local" + port = 32010 + default = true + # Federation OUTGOING auth to this sidecar — single source of truth with the + # sidecar's ARROW_AUTH_* env. For a Secret-backed credential (credentialsSecretName) + # the VALUE is absent here and arrives via CONFIG_FORCE_* (override_with_env_vars) on + # the federation Deployment (Story 16.3, FACT A) — this block then renders method-only, + # and the override forces the value before FederationConfig.validate() runs. Inline + # values (dev/test) are rendered when present. + credentials { + method = "bearer" + } + } + } + } +--- +# Source: softclient4es-federation/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32020 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed +--- +# Source: softclient4es-federation/templates/sidecar-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation-prod-us + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-us + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32010 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-us +--- +# Source: softclient4es-federation/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + annotations: + # Roll the federation Pod whenever the rendered servers ConfigMap changes + # (add/remove/edit a sidecar) so `helm upgrade` actually takes effect. + checksum/config: 96ea5304467e9aeffe1259c56d05d5c9093a1d7c3d1611bf4315d75697e47dbe + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: federation + image: "docker.io/softnetwork/softclient4es-federation:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32020 + protocol: TCP + - name: health + containerPort: 32021 + protocol: TCP + env: + - name: FEDERATION_HOST + value: "0.0.0.0" + - name: FEDERATION_PORT + value: "32020" + - name: FEDERATION_MAX_MEMORY + value: "512m" + - name: FEDERATION_QUERY_TIMEOUT + value: "30" + - name: FEDERATION_HEALTH_PORT + value: "32021" + - name: FEDERATION_HEALTH_PROBE_TIMEOUT + value: "5" + - name: FEDERATION_DUCKDB_PATH + value: ":memory:" + - name: FEDERATION_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + # Point Typesafe Config at the mounted ConfigMap so ConfigFactory.load() + # picks up the `servers` map (which has NO env override). The native-packager + # bash launcher reads $JAVA_OPTS at runtime (VERIFIED) and prepends it to the + # JVM args; reference.conf still loads, so the FEDERATION_* env defaults stay. + # Story 16.3: when any sidecar uses Secret-backed auth, also enable + # override_with_env_vars so the CONFIG_FORCE_* env below force the credential + # leaf onto the loaded config BEFORE FederationConfig.validate() runs. + - name: JAVA_OPTS + value: "-Dconfig.file=/opt/docker/conf/application.conf -Dconfig.override_with_env_vars=true" + - name: CONFIG_FORCE_arrow_flight_federation_servers_prod__us_credentials_bearer__token + valueFrom: + secretKeyRef: + name: prod-us-arrow-auth + key: arrow-bearer-token + optional: true + # --- Probes (FACT A / A1): TCP-socket only for the skeleton. --- + # The federation health endpoint is gRPC grpc.health.v1.Health and returns + # NOT_SERVING while servers={} (HealthService.scala:105-111), so a gRPC Check + # probe would fail. A TCP probe proves the process is listening, which is the + # correct liveness/readiness signal for a no-downstream skeleton. Story 16.2 + # switches readiness to a gRPC probe once sidecars exist. + livenessProbe: + tcpSocket: + port: health + initialDelaySeconds: 20 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + # Readiness flips to a native gRPC probe once sidecars make the SERVING + # aggregate meaningful (Story 16.2). ⚠️ ALL-OR-NOTHING: any one unreachable + # downstream → NotReady → ALL queries fail (fail-closed). Set + # federation.probes.useGrpc=false to keep TCP readiness (partial availability; + # required on K8s < 1.27 — native gRPC probes are GA only from 1.27). + readinessProbe: + grpc: + port: 32021 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + volumeMounts: + - name: scratch + mountPath: /tmp + - name: federation-config + mountPath: /opt/docker/conf + readOnly: true + volumes: + # readOnlyRootFilesystem=true requires a writable scratch dir: the org.duckdb + # JDBC driver extracts its native libduckdb_java library to java.io.tmpdir (/tmp) + # on every boot (PersistentDuckDB is instantiated unconditionally — even with + # path=":memory:"), and the federation stages CTAS / INSERT-with-JOIN Parquet + # under java.io.tmpdir (FederationConfig.scala:266-267 → /tmp/softclient4es/scratch). + # A file DuckDB path (federation.duckdb.path != ":memory:") would also live here. + - name: scratch + emptyDir: + sizeLimit: 2Gi + - name: federation-config + configMap: + name: fed-softclient4es-federation-config +--- +# Source: softclient4es-federation/templates/sidecar-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation-prod-us + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-us + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-us + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-us + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: sidecar + image: "docker.io/softnetwork/softclient4es8-arrow-flight-sql:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32010 + protocol: TCP + env: + # ── Arrow Flight SQL server (arrow.flight.*) ── + - name: ARROW_HOST + value: "0.0.0.0" + - name: ARROW_PORT + value: "32010" + - name: ARROW_BATCH_SIZE + value: "1000" + - name: ARROW_QUERY_TIMEOUT_SECONDS + value: "120" + - name: ARROW_JOIN_MAX_MEMORY + value: "256m" + - name: ARROW_JOIN_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + # ── Elasticsearch connection (elastic.credentials.* — VERIFIED, NOT a single URL) ── + - name: ELASTIC_SCHEME + value: "https" + - name: ELASTIC_HOST + value: "es-us.example.com" + - name: ELASTIC_PORT + value: "9200" + - name: ELASTIC_AUTH_METHOD + valueFrom: + secretKeyRef: + name: es-prod-us + key: es-auth-method + optional: true + - name: ELASTIC_CREDENTIALS_USERNAME + valueFrom: + secretKeyRef: + name: es-prod-us + key: es-username + optional: true + - name: ELASTIC_CREDENTIALS_PASSWORD + valueFrom: + secretKeyRef: + name: es-prod-us + key: es-password + optional: true + - name: ELASTIC_CREDENTIALS_API_KEY + valueFrom: + secretKeyRef: + name: es-prod-us + key: es-api-key + optional: true + - name: ELASTIC_CREDENTIALS_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: es-prod-us + key: es-bearer-token + optional: true + # ── Sidecar SERVER-SIDE auth (incoming) — single source of truth with the + # federation's outgoing servers..credentials (rendered in the ConfigMap) ── + - name: ARROW_AUTH_METHOD + value: "bearer" + - name: ARROW_AUTH_USERNAME + valueFrom: + secretKeyRef: + name: prod-us-arrow-auth + key: arrow-username + optional: true + - name: ARROW_AUTH_PASSWORD + valueFrom: + secretKeyRef: + name: prod-us-arrow-auth + key: arrow-password + optional: true + - name: ARROW_AUTH_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: prod-us-arrow-auth + key: arrow-bearer-token + optional: true + - name: ARROW_AUTH_API_KEY + valueFrom: + secretKeyRef: + name: prod-us-arrow-auth + key: arrow-api-key + optional: true + # ── Telemetry opt-out (shared with federation) ── + # VERIFIED: SOFTCLIENT4ES_TELEMETRY_ENABLED is honored by the sidecar too — the env + # override lives in elasticsql licensing reference.conf:30 (softclient4es.telemetry.enabled), + # read by TelemetryConfig.load, on which the arrow sidecar depends transitively. NOTE it is + # NOT in arrowServerSettings.dockerEnvVars (build.sbt:243-255 bakes only ELASTIC_*/ARROW_*), + # so the chart MUST set it explicitly here for the opt-out to reach the sidecar. + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + # TCP probes — the sidecar has NO gRPC health service (VERIFIED); a raw TCP + # connect to ARROW_PORT proves the Flight SQL server is listening. + livenessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 20 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 1536Mi + requests: + cpu: 500m + memory: 768Mi + volumeMounts: + - name: scratch + mountPath: /tmp + volumes: + # Writable /tmp: the sidecar runs the same-cluster JOIN engine on DuckDB + # (DuckDBJoinExecutor via LocalJoinHandler — VERIFIED), whose org.duckdb JDBC + # driver extracts its native libduckdb_java to java.io.tmpdir on first JOIN. + # readOnlyRootFilesystem=true without this would crash the first JOIN query. + - name: scratch + emptyDir: + sizeLimit: 1Gi +--- +# Source: softclient4es-federation/templates/tests/smoke-test.yaml +apiVersion: v1 +kind: Pod +metadata: + name: fed-softclient4es-federation-test-catalogs + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: show-catalogs + # Default: a public python image; CI + air-gapped clusters MUST pin `test.image` + # to a PRE-BAKED image with adbc_driver_flightsql installed (no runtime pip). + image: "python:3.12-slim" + command: ["sh", "-c"] + args: + - | + set -e + # If the image is NOT pre-baked, install ADBC at runtime (needs PyPI reachability). + python -c "import adbc_driver_flightsql" 2>/dev/null || \ + pip install --quiet adbc_driver_flightsql==1.6.0 \ + adbc_driver_manager==1.6.0 + python - <<'PY' + import sys, time + import adbc_driver_flightsql.dbapi as dbapi + host = "fed-softclient4es-federation" + uri = f"grpc://{host}:32020" + expected = 1 + # The federation only reports its catalogs once it has DISCOVERED every downstream + # (gRPC readiness is all-or-nothing). Retry the connect + GetCatalogs for up to ~60s. + deadline = time.time() + 60 + last_err = None + while time.time() < deadline: + try: + conn = dbapi.connect(uri) + # GetCatalogs: one catalog_name row per registered sidecar alias (VERIFIED + # FederationFlightProducer.getStreamCatalogs — needs NO default downstream). + cats = [c["catalog_name"] + for c in conn.adbc_get_objects(depth="catalogs").read_all().to_pylist()] + cats = [c for c in cats if c] # drop empty/system catalog if present + conn.close() + if len(cats) == expected: + print(f"OK: {len(cats)} catalogs: {cats}") + sys.exit(0) + last_err = f"expected {expected} catalogs, got {len(cats)}: {cats}" + except Exception as e: # connect refused while federation still NotReady + last_err = f"connect/get_objects error: {e}" + time.sleep(3) + print(f"FAIL: {last_err}", file=sys.stderr) + sys.exit(1) + PY diff --git a/softclient4es-federation/tests/golden/two-sidecars.yaml b/softclient4es-federation/tests/golden/two-sidecars.yaml new file mode 100644 index 0000000..8353c81 --- /dev/null +++ b/softclient4es-federation/tests/golden/two-sidecars.yaml @@ -0,0 +1,654 @@ +--- +# Source: softclient4es-federation/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +automountServiceAccountToken: true +--- +# Source: softclient4es-federation/templates/federation-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: fed-softclient4es-federation-config + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +data: + application.conf: | + # GENERATED by the softclient4es-federation Helm chart — do not edit in-cluster. + # Populates arrow.flight.federation.servers from values.yaml `sidecars[]`. + # The scalar FEDERATION_* settings are still supplied via env vars on the + # Deployment; this file ONLY carries the `servers` map (which has no env override). + arrow.flight.federation { + servers { + prod-us { + host = "fed-softclient4es-federation-prod-us.default.svc.cluster.local" + port = 32010 + default = true + } + prod-eu { + host = "fed-softclient4es-federation-prod-eu.default.svc.cluster.local" + port = 32010 + } + } + } +--- +# Source: softclient4es-federation/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32020 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed +--- +# Source: softclient4es-federation/templates/sidecar-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation-prod-us + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-us + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32010 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-us +--- +# Source: softclient4es-federation/templates/sidecar-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fed-softclient4es-federation-prod-eu + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-eu + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + type: ClusterIP + ports: + - name: flight-sql + port: 32010 + targetPort: flight-sql + protocol: TCP + selector: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-eu +--- +# Source: softclient4es-federation/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + annotations: + # Roll the federation Pod whenever the rendered servers ConfigMap changes + # (add/remove/edit a sidecar) so `helm upgrade` actually takes effect. + checksum/config: 24fa1a5267bbb746f19103e9801a86915f61bb22678cde9321901bf3ec41c2fb + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: federation + image: "docker.io/softnetwork/softclient4es-federation:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32020 + protocol: TCP + - name: health + containerPort: 32021 + protocol: TCP + env: + - name: FEDERATION_HOST + value: "0.0.0.0" + - name: FEDERATION_PORT + value: "32020" + - name: FEDERATION_MAX_MEMORY + value: "512m" + - name: FEDERATION_QUERY_TIMEOUT + value: "30" + - name: FEDERATION_HEALTH_PORT + value: "32021" + - name: FEDERATION_HEALTH_PROBE_TIMEOUT + value: "5" + - name: FEDERATION_DUCKDB_PATH + value: ":memory:" + - name: FEDERATION_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + - name: SOFTCLIENT4ES_LICENSE_KEY + valueFrom: + secretKeyRef: + name: sc4es-license + key: license-key + optional: true + - name: SOFTCLIENT4ES_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-license + key: api-key + optional: true + # Point Typesafe Config at the mounted ConfigMap so ConfigFactory.load() + # picks up the `servers` map (which has NO env override). The native-packager + # bash launcher reads $JAVA_OPTS at runtime (VERIFIED) and prepends it to the + # JVM args; reference.conf still loads, so the FEDERATION_* env defaults stay. + # Story 16.3: when any sidecar uses Secret-backed auth, also enable + # override_with_env_vars so the CONFIG_FORCE_* env below force the credential + # leaf onto the loaded config BEFORE FederationConfig.validate() runs. + - name: JAVA_OPTS + value: "-Dconfig.file=/opt/docker/conf/application.conf" + # --- Probes (FACT A / A1): TCP-socket only for the skeleton. --- + # The federation health endpoint is gRPC grpc.health.v1.Health and returns + # NOT_SERVING while servers={} (HealthService.scala:105-111), so a gRPC Check + # probe would fail. A TCP probe proves the process is listening, which is the + # correct liveness/readiness signal for a no-downstream skeleton. Story 16.2 + # switches readiness to a gRPC probe once sidecars exist. + livenessProbe: + tcpSocket: + port: health + initialDelaySeconds: 20 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 3 + # Readiness flips to a native gRPC probe once sidecars make the SERVING + # aggregate meaningful (Story 16.2). ⚠️ ALL-OR-NOTHING: any one unreachable + # downstream → NotReady → ALL queries fail (fail-closed). Set + # federation.probes.useGrpc=false to keep TCP readiness (partial availability; + # required on K8s < 1.27 — native gRPC probes are GA only from 1.27). + readinessProbe: + grpc: + port: 32021 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + volumeMounts: + - name: scratch + mountPath: /tmp + - name: federation-config + mountPath: /opt/docker/conf + readOnly: true + volumes: + # readOnlyRootFilesystem=true requires a writable scratch dir: the org.duckdb + # JDBC driver extracts its native libduckdb_java library to java.io.tmpdir (/tmp) + # on every boot (PersistentDuckDB is instantiated unconditionally — even with + # path=":memory:"), and the federation stages CTAS / INSERT-with-JOIN Parquet + # under java.io.tmpdir (FederationConfig.scala:266-267 → /tmp/softclient4es/scratch). + # A file DuckDB path (federation.duckdb.path != ":memory:") would also live here. + - name: scratch + emptyDir: + sizeLimit: 2Gi + - name: federation-config + configMap: + name: fed-softclient4es-federation-config +--- +# Source: softclient4es-federation/templates/sidecar-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation-prod-us + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-us + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-us + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-us + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: sidecar + image: "docker.io/softnetwork/softclient4es8-arrow-flight-sql:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32010 + protocol: TCP + env: + # ── Arrow Flight SQL server (arrow.flight.*) ── + - name: ARROW_HOST + value: "0.0.0.0" + - name: ARROW_PORT + value: "32010" + - name: ARROW_BATCH_SIZE + value: "1000" + - name: ARROW_QUERY_TIMEOUT_SECONDS + value: "120" + - name: ARROW_JOIN_MAX_MEMORY + value: "256m" + - name: ARROW_JOIN_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + # ── Elasticsearch connection (elastic.credentials.* — VERIFIED, NOT a single URL) ── + - name: ELASTIC_SCHEME + value: "https" + - name: ELASTIC_HOST + value: "es-us.example.com" + - name: ELASTIC_PORT + value: "9200" + - name: ELASTIC_AUTH_METHOD + valueFrom: + secretKeyRef: + name: es-us-creds + key: es-auth-method + optional: true + - name: ELASTIC_CREDENTIALS_USERNAME + valueFrom: + secretKeyRef: + name: es-us-creds + key: es-username + optional: true + - name: ELASTIC_CREDENTIALS_PASSWORD + valueFrom: + secretKeyRef: + name: es-us-creds + key: es-password + optional: true + - name: ELASTIC_CREDENTIALS_API_KEY + valueFrom: + secretKeyRef: + name: es-us-creds + key: es-api-key + optional: true + - name: ELASTIC_CREDENTIALS_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: es-us-creds + key: es-bearer-token + optional: true + # ── Sidecar SERVER-SIDE auth (incoming) — single source of truth with the + # federation's outgoing servers..credentials (rendered in the ConfigMap) ── + # ── Telemetry opt-out (shared with federation) ── + # VERIFIED: SOFTCLIENT4ES_TELEMETRY_ENABLED is honored by the sidecar too — the env + # override lives in elasticsql licensing reference.conf:30 (softclient4es.telemetry.enabled), + # read by TelemetryConfig.load, on which the arrow sidecar depends transitively. NOTE it is + # NOT in arrowServerSettings.dockerEnvVars (build.sbt:243-255 bakes only ELASTIC_*/ARROW_*), + # so the chart MUST set it explicitly here for the opt-out to reach the sidecar. + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + - name: SOFTCLIENT4ES_LICENSE_KEY + valueFrom: + secretKeyRef: + name: sc4es-license + key: license-key + optional: true + - name: SOFTCLIENT4ES_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-license + key: api-key + optional: true + # TCP probes — the sidecar has NO gRPC health service (VERIFIED); a raw TCP + # connect to ARROW_PORT proves the Flight SQL server is listening. + livenessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 20 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 1536Mi + requests: + cpu: 500m + memory: 768Mi + volumeMounts: + - name: scratch + mountPath: /tmp + volumes: + # Writable /tmp: the sidecar runs the same-cluster JOIN engine on DuckDB + # (DuckDBJoinExecutor via LocalJoinHandler — VERIFIED), whose org.duckdb JDBC + # driver extracts its native libduckdb_java to java.io.tmpdir on first JOIN. + # readOnlyRootFilesystem=true without this would crash the first JOIN query. + - name: scratch + emptyDir: + sizeLimit: 1Gi +--- +# Source: softclient4es-federation/templates/sidecar-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fed-softclient4es-federation-prod-eu + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-eu + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: softclient4es +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-eu + template: + metadata: + labels: + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/component: sidecar + softclient4es.app/sidecar: prod-eu + spec: + serviceAccountName: fed-softclient4es-federation + securityContext: + fsGroup: 1001 + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: sidecar + image: "docker.io/softnetwork/softclient4es9-arrow-flight-sql:0.2.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + ports: + - name: flight-sql + containerPort: 32010 + protocol: TCP + env: + # ── Arrow Flight SQL server (arrow.flight.*) ── + - name: ARROW_HOST + value: "0.0.0.0" + - name: ARROW_PORT + value: "32010" + - name: ARROW_BATCH_SIZE + value: "1000" + - name: ARROW_QUERY_TIMEOUT_SECONDS + value: "120" + - name: ARROW_JOIN_MAX_MEMORY + value: "256m" + - name: ARROW_JOIN_UPGRADE_URL + value: "https://portal.softclient4es.com/pricing" + # ── Elasticsearch connection (elastic.credentials.* — VERIFIED, NOT a single URL) ── + - name: ELASTIC_SCHEME + value: "https" + - name: ELASTIC_HOST + value: "es-eu.example.com" + - name: ELASTIC_PORT + value: "9200" + - name: ELASTIC_AUTH_METHOD + valueFrom: + secretKeyRef: + name: es-eu-creds + key: es-auth-method + optional: true + - name: ELASTIC_CREDENTIALS_USERNAME + valueFrom: + secretKeyRef: + name: es-eu-creds + key: es-username + optional: true + - name: ELASTIC_CREDENTIALS_PASSWORD + valueFrom: + secretKeyRef: + name: es-eu-creds + key: es-password + optional: true + - name: ELASTIC_CREDENTIALS_API_KEY + valueFrom: + secretKeyRef: + name: es-eu-creds + key: es-api-key + optional: true + - name: ELASTIC_CREDENTIALS_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: es-eu-creds + key: es-bearer-token + optional: true + # ── Sidecar SERVER-SIDE auth (incoming) — single source of truth with the + # federation's outgoing servers..credentials (rendered in the ConfigMap) ── + # ── Telemetry opt-out (shared with federation) ── + # VERIFIED: SOFTCLIENT4ES_TELEMETRY_ENABLED is honored by the sidecar too — the env + # override lives in elasticsql licensing reference.conf:30 (softclient4es.telemetry.enabled), + # read by TelemetryConfig.load, on which the arrow sidecar depends transitively. NOTE it is + # NOT in arrowServerSettings.dockerEnvVars (build.sbt:243-255 bakes only ELASTIC_*/ARROW_*), + # so the chart MUST set it explicitly here for the opt-out to reach the sidecar. + - name: SOFTCLIENT4ES_TELEMETRY_ENABLED + value: "true" + - name: SOFTCLIENT4ES_LICENSE_KEY + valueFrom: + secretKeyRef: + name: sc4es-license + key: license-key + optional: true + - name: SOFTCLIENT4ES_API_KEY + valueFrom: + secretKeyRef: + name: sc4es-license + key: api-key + optional: true + # TCP probes — the sidecar has NO gRPC health service (VERIFIED); a raw TCP + # connect to ARROW_PORT proves the Flight SQL server is listening. + livenessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 20 + periodSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: flight-sql + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + resources: + limits: + cpu: 1000m + memory: 1536Mi + requests: + cpu: 500m + memory: 768Mi + volumeMounts: + - name: scratch + mountPath: /tmp + volumes: + # Writable /tmp: the sidecar runs the same-cluster JOIN engine on DuckDB + # (DuckDBJoinExecutor via LocalJoinHandler — VERIFIED), whose org.duckdb JDBC + # driver extracts its native libduckdb_java to java.io.tmpdir on first JOIN. + # readOnlyRootFilesystem=true without this would crash the first JOIN query. + - name: scratch + emptyDir: + sizeLimit: 1Gi +--- +# Source: softclient4es-federation/templates/tests/smoke-test.yaml +apiVersion: v1 +kind: Pod +metadata: + name: fed-softclient4es-federation-test-catalogs + labels: + helm.sh/chart: softclient4es-federation-0.3.0 + app.kubernetes.io/name: softclient4es-federation + app.kubernetes.io/instance: fed + app.kubernetes.io/version: "0.2.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: federation + app.kubernetes.io/part-of: softclient4es + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + restartPolicy: Never + containers: + - name: show-catalogs + # Default: a public python image; CI + air-gapped clusters MUST pin `test.image` + # to a PRE-BAKED image with adbc_driver_flightsql installed (no runtime pip). + image: "python:3.12-slim" + command: ["sh", "-c"] + args: + - | + set -e + # If the image is NOT pre-baked, install ADBC at runtime (needs PyPI reachability). + python -c "import adbc_driver_flightsql" 2>/dev/null || \ + pip install --quiet adbc_driver_flightsql==1.6.0 \ + adbc_driver_manager==1.6.0 + python - <<'PY' + import sys, time + import adbc_driver_flightsql.dbapi as dbapi + host = "fed-softclient4es-federation" + uri = f"grpc://{host}:32020" + expected = 2 + # The federation only reports its catalogs once it has DISCOVERED every downstream + # (gRPC readiness is all-or-nothing). Retry the connect + GetCatalogs for up to ~60s. + deadline = time.time() + 60 + last_err = None + while time.time() < deadline: + try: + conn = dbapi.connect(uri) + # GetCatalogs: one catalog_name row per registered sidecar alias (VERIFIED + # FederationFlightProducer.getStreamCatalogs — needs NO default downstream). + cats = [c["catalog_name"] + for c in conn.adbc_get_objects(depth="catalogs").read_all().to_pylist()] + cats = [c for c in cats if c] # drop empty/system catalog if present + conn.close() + if len(cats) == expected: + print(f"OK: {len(cats)} catalogs: {cats}") + sys.exit(0) + last_err = f"expected {expected} catalogs, got {len(cats)}: {cats}" + except Exception as e: # connect refused while federation still NotReady + last_err = f"connect/get_objects error: {e}" + time.sleep(3) + print(f"FAIL: {last_err}", file=sys.stderr) + sys.exit(1) + PY diff --git a/softclient4es-federation/tests/values/ingress-tls.yaml b/softclient4es-federation/tests/values/ingress-tls.yaml new file mode 100644 index 0000000..7936984 --- /dev/null +++ b/softclient4es-federation/tests/values/ingress-tls.yaml @@ -0,0 +1,23 @@ +# tests/values/ingress-tls.yaml — federation behind a cert-manager TLS Ingress. +# Single sidecar (license-free) so this render also stays unconditional in CI. +federation: + tls: + enabled: true + secretName: fed-tls +ingress: + enabled: true + className: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/backend-protocol: "GRPC" + hosts: + - host: fed.example.com + paths: + - path: / + pathType: Prefix +sidecars: + - name: prod-us + elasticsearchVersion: 8 + elasticsearch: + url: "https://es-us.example.com:9200" + default: true diff --git a/softclient4es-federation/tests/values/secret-auth.yaml b/softclient4es-federation/tests/values/secret-auth.yaml new file mode 100644 index 0000000..da68188 --- /dev/null +++ b/softclient4es-federation/tests/values/secret-auth.yaml @@ -0,0 +1,14 @@ +# tests/values/secret-auth.yaml — 1 sidecar, Secret-backed ES + sidecar bearer auth. +# License-free (1 sidecar, Community maxClusters=1 — FACT D), so CI runs it unconditionally. +# Exercises: ES creds via secretKeyRef, sidecar ARROW_AUTH_* via secretKeyRef, and the +# federation CONFIG_FORCE_* override path (override_with_env_vars) end-to-end. +sidecars: + - name: prod-us + elasticsearchVersion: 8 + elasticsearch: + url: "https://es-us.example.com:9200" + credentialsSecretName: es-prod-us + auth: + method: bearer + credentialsSecretName: prod-us-arrow-auth # feeds sidecar ARROW_AUTH_* + federation CONFIG_FORCE_* + default: true diff --git a/softclient4es-federation/tests/values/two-sidecars.yaml b/softclient4es-federation/tests/values/two-sidecars.yaml new file mode 100644 index 0000000..441e330 --- /dev/null +++ b/softclient4es-federation/tests/values/two-sidecars.yaml @@ -0,0 +1,17 @@ +# Golden-file input: federation + 2 sidecars on DIFFERENT ES versions (the epic's +# mixed-version requirement). Used by `helm template -f` to regen +# tests/golden/two-sidecars.yaml. Keep entries in a FIXED order for golden stability. +license: + secretName: sc4es-license # Pro/Enterprise — required for >=2 sidecars (Community maxClusters=1, FACT D) +sidecars: + - name: prod-us + elasticsearchVersion: 8 + elasticsearch: + url: "https://es-us.example.com:9200" + credentialsSecretName: es-us-creds + default: true + - name: prod-eu + elasticsearchVersion: 9 + elasticsearch: + url: "https://es-eu.example.com:9200" + credentialsSecretName: es-eu-creds diff --git a/softclient4es-federation/values.yaml b/softclient4es-federation/values.yaml new file mode 100644 index 0000000..07af596 --- /dev/null +++ b/softclient4es-federation/values.yaml @@ -0,0 +1,334 @@ +# Number of federation Pods. Default 1 — the federation is stateless wrt query +# routing; HA scale-out is documented at >=2 in the operator guide (16.6). +replicaCount: 1 + +image: + repository: docker.io/softnetwork/softclient4es-federation + # Defaults to .Chart.AppVersion when empty; override for pinning or local images. + tag: "" + pullPolicy: IfNotPresent + +# For authenticated DockerHub pulls (rate-limit mitigation). List of {name: }. +imagePullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + name: "" + annotations: {} + automount: true + +# Federation config — maps 1:1 onto arrow.flight.federation.* (FEDERATION_* env). +# Defaults VERIFIED against federation/src/main/resources/reference.conf. +federation: + maxMemory: "512m" # FEDERATION_MAX_MEMORY (DuckDB SET memory_limit) + queryTimeoutSeconds: 30 # FEDERATION_QUERY_TIMEOUT + health: + port: 32021 # FEDERATION_HEALTH_PORT (gRPC grpc.health.v1.Health) + probeTimeoutSeconds: 5 # FEDERATION_HEALTH_PROBE_TIMEOUT (also K8s probe timeoutSeconds) + duckdb: + # ":memory:" = in-memory only (default). A file path enables a persistent + # DuckDB catalog — R2b will recommend a path on a PersistentVolume. A file + # path under readOnlyRootFilesystem=true MUST be on a writable mount (e.g. + # under /tmp, which is the writable scratch emptyDir, or a PersistentVolume). + path: ":memory:" # FEDERATION_DUCKDB_PATH + upgradeUrl: "https://portal.softclient4es.com/pricing" # FEDERATION_UPGRADE_URL + # Federation probe mode (Story 16.2). 16.1 used TCP-socket probes because a gRPC + # health Check returns NOT_SERVING while servers={}. Once sidecars exist the gRPC + # SERVING aggregate becomes meaningful, so 16.2 can flip the READINESS probe to a + # native gRPC probe. + probes: + # When true (default) AND at least one sidecar is configured, the federation + # READINESS probe is a native gRPC probe (grpc.health.v1.Health on 32021). + # ⚠️ The SERVING aggregate is ALL-OR-NOTHING (VERIFIED HealthService.scala:154-169: + # `if (failures.isEmpty) SERVING else NOT_SERVING`): if ANY ONE downstream sidecar + # is unreachable the federation Pod goes NotReady and is pulled from its Service — + # so EVERY federation query fails, including ones targeting still-healthy sidecars + # ("fail-closed" routing). Requires Kubernetes >= 1.27 (GA gRPC probes). + # Set false to keep the 16.1 TCP readiness (no K8s version floor; partial availability — + # the federation stays Ready and degrades per-query). On K8s 1.24-1.26 you MUST set + # false (use the grpc_health_probe exec fallback — see the operator guide). Liveness + # always stays TCP (a transient downstream blip must not kill the federation process). + useGrpc: true + # ── TLS for the federation's PUBLIC Flight SQL endpoint (Story 16.3) ────────── + # IMPORTANT (VERIFIED): the federation server listens PLAINTEXT gRPC only + # (FederationFlightServer.scala:93 Location.forGrpcInsecure; banner "plaintext only"). + # It CANNOT terminate TLS at the pod. So TLS is terminated at an INGRESS / gateway. + # Enable `ingress.enabled` + set `federation.tls.secretName` to a cert-manager-issued + # kubernetes.io/tls Secret (tls.crt + tls.key). The chart does NOT create this Secret. + tls: + # When true, the Ingress (below) gets a tls: block referencing `secretName`. + # Pod-level TLS is NOT supported by the federation server (plaintext only). + enabled: false + # A kubernetes.io/tls Secret (tls.crt + tls.key), e.g. issued by cert-manager. + secretName: "" + # ── Secret-backed federation→sidecar OUTGOING credentials (Story 16.3) ──────── + # Resolves 16.2 OQ-2. When true (default) AND a sidecar uses a Secret-backed auth + # method, the federation Deployment sets -Dconfig.override_with_env_vars=true and + # injects CONFIG_FORCE_arrow_flight_federation_servers__credentials_ env + # from the SAME Secret the sidecar reads — so the credential never lives in the + # ConfigMap. Requires Typesafe Config >= 1.4.0 (the federation's transitive config + # lib; VERIFIED >= 1.4.2 via Akka 2.6.20). Set false to require inline creds instead; + # a Secret-backed sidecar auth with credentialsFromEnv=false then fails at template + # time (the federation cannot read the Secret). + credentialsFromEnv: true + +# ───────────────────────────────────────────────────────────────────────────── +# Ingress for the federation Flight SQL endpoint (Story 16.3). OPTIONAL. +# +# Flight SQL is gRPC: the Ingress controller MUST support gRPC backends, e.g. nginx: +# nginx.ingress.kubernetes.io/backend-protocol: "GRPC" +# (or use a Gateway-API / Envoy / Contour gateway). A default HTTP-only Ingress will +# NOT proxy Flight SQL. `annotations` is free-form — populate it with your +# cert-manager / external-DNS / ingress-controller preferences. +ingress: + enabled: false + className: "" # e.g. "nginx" (spec.ingressClassName) + annotations: {} + # cert-manager.io/cluster-issuer: letsencrypt-prod + # nginx.ingress.kubernetes.io/backend-protocol: "GRPC" + # external-dns.alpha.kubernetes.io/hostname: fed.example.com + hosts: + - host: "" # e.g. fed.example.com (empty default = no rule rendered) + paths: + - path: / + pathType: Prefix + # TLS block for the Ingress. If left empty AND federation.tls.enabled, the chart + # auto-fills a single entry from federation.tls.secretName + ingress.hosts[].host. + tls: [] + # - secretName: fed-tls # kubernetes.io/tls (cert-manager issues this) + # hosts: + # - fed.example.com + +# Daily product-instance telemetry ping opt-out (Story 15.2/15.6). Top-level +# softclient4es.telemetry.enabled → env SOFTCLIENT4ES_TELEMETRY_ENABLED. +# DISTINCT from license.telemetry.enabled (refresh-metrics only). Set false to opt out. +telemetry: + enabled: true + +# License credentials are referenced from a customer-provided K8s Secret (the chart +# does NOT create Secrets — Story 16.3 covers secret-management patterns). Leave +# secretName empty to run unlicensed (Community). +license: + secretName: "" + licenseKeyKey: license-key # Secret data key holding the JWT license -> SOFTCLIENT4ES_LICENSE_KEY + apiKeyKey: api-key # Secret data key holding the API key -> SOFTCLIENT4ES_API_KEY + # ── Offline license public-key (Story 16.5, FACT F #2) ──────────────────────── + # OPTIONAL. When the federation must verify a license JWT WITHOUT reaching the + # license server's JWKS endpoint (air-gapped clusters, or a test-signed JWT whose + # `kid` is not in the prod JWKS), point publicKeySecretName at a Secret whose + # `publicKeyKey` data key holds the matching Ed25519 public JWK. It is mounted as + # SOFTCLIENT4ES_LICENSE_PUBLIC_KEY — the air-gap path in LicenseKeyVerifier.loadPublicKey + # (Step 3, VERIFIED). Leave empty (the default) to use the normal JWKS fetch. Renders + # NOTHING when empty, so the golden render and every shipped example are unaffected. + publicKeySecretName: "" + publicKeyKey: license-public-key # Secret data key holding the Ed25519 public JWK -> SOFTCLIENT4ES_LICENSE_PUBLIC_KEY + +service: + type: ClusterIP + port: 32020 # FEDERATION_PORT (Flight SQL) + +# Probes are TCP-socket (FACT A / A1). timeoutSeconds is taken from +# federation.health.probeTimeoutSeconds in the Deployment template. +probes: + liveness: + initialDelaySeconds: 20 + periodSeconds: 15 + failureThreshold: 3 + readiness: + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 # aggressive for fast no-traffic routing per epic AC + +# Resource defaults sized for max-memory=512m DuckDB + JVM heap + Arrow off-heap +# + DuckDB native. See Architecture Decision A4 for the math. +resources: + requests: + memory: 1Gi + cpu: 500m + limits: + memory: 2Gi + cpu: 1000m + +# Writable scratch volume (emptyDir) mounted at /tmp so readOnlyRootFilesystem=true +# still allows the DuckDB JNI native-lib extraction on boot + CTAS/INSERT-JOIN +# Parquet staging + any non-:memory: DuckDB file. +scratch: + sizeLimit: 2Gi + +# Pod-level security context (runAsNonRoot etc.). +podSecurityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + seccompProfile: + type: RuntimeDefault + +# Container-level security context (hardening). +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + capabilities: + drop: + - ALL + +nodeSelector: {} +tolerations: [] +affinity: {} + +# ───────────────────────────────────────────────────────────────────────────── +# Per-Elasticsearch-version sidecars (Story 16.2). +# +# Each entry deploys ONE Arrow Flight SQL gateway Deployment + Service in front of +# ONE Elasticsearch cluster, and registers that cluster in the federation's +# `arrow.flight.federation.servers.` map (rendered into a ConfigMap). +# +# LICENSING GUARDRAIL (VERIFIED — JoinLicenseGuard.guardStartup, Step 2 maxClusters): +# * Community (no license) permits EXACTLY ONE sidecar (maxClusters=1) — a +# 1-sidecar deployment boots Ready with no license (the free single-cluster tier). +# * TWO OR MORE sidecars (len(sidecars) >= 2) require a Pro or Enterprise license +# — set `license.secretName` above. With NO/Community license and >=2 sidecars +# the federation Pod CrashLoops by design (sys.exit on the startup quota guard). +# * Pro allows up to 5 sidecars (maxClusters=5); Enterprise = unlimited. +# * NB: the Federation FEATURE is present in every tier (incl. Community); the gate +# is the per-tier maxClusters QUOTA, not a feature flag. +# +# Default: empty — the chart behaves exactly like 16.1 (federation only) when +# `sidecars` is []. +sidecars: [] +# - name: prod-us # logical name: federation servers. key AND k8s resource suffix +# # (must be RFC1123: lowercase alphanumeric + '-', unique) +# elasticsearchVersion: 8 # 6 | 7 | 8 | 9 -> selects the sidecar image +# elasticsearch: +# # The backing ES cluster. `url` is DECOMPOSED into ELASTIC_SCHEME/HOST/PORT +# # (there is NO single ES-URL env var — VERIFIED). Either give `url`, or set +# # scheme/host/port explicitly (explicit fields win over url-derived ones). +# # `url` must be the canonical `scheme://host:port` form (no path/userinfo) — +# # a scheme-less url defaults to http, a port-less url to 9200; for a TLS or +# # non-9200 cluster supply the scheme/port or use explicit fields. +# url: "https://es-prod-us.example.com:9200" +# # scheme: https # optional explicit override -> ELASTIC_SCHEME +# # host: es-prod-us... # optional explicit override -> ELASTIC_HOST +# # port: 9200 # optional explicit override -> ELASTIC_PORT +# # K8s Secret holding ES credentials. Keys (any subset): es-auth-method, +# # es-username, es-password, es-api-key, es-bearer-token. Mounted as +# # ELASTIC_AUTH_METHOD / ELASTIC_CREDENTIALS_* env. Chart does NOT create it. +# credentialsSecretName: "" +# # Optional (Story 16.3): override the Secret data-key names the chart reads +# # via secretKeyRef.key (defaults are the FACT-C contract keys shown). Use this +# # when your ESO/Vault template emits different key names. +# secretKeys: +# authMethod: es-auth-method # -> ELASTIC_AUTH_METHOD +# username: es-username # -> ELASTIC_CREDENTIALS_USERNAME +# password: es-password # -> ELASTIC_CREDENTIALS_PASSWORD +# apiKey: es-api-key # -> ELASTIC_CREDENTIALS_API_KEY +# bearerToken: es-bearer-token # -> ELASTIC_CREDENTIALS_BEARER_TOKEN +# # Optional (Story 16.3): mount the WHOLE Secret as env (envFrom.secretRef) +# # instead of per-key. The Secret's data keys must then BE the env-var names +# # (ELASTIC_*); there is no key remapping in this mode. Default false. +# useEnvFrom: false +# replicaCount: 1 # default 1; HA = 3 (documented in README) +# resources: {} # optional per-sidecar override of sidecarDefaults.resources +# arrow: +# maxMemory: "256m" # ARROW_JOIN_MAX_MEMORY (DuckDB JOIN engine memory) +# queryTimeoutSeconds: 120 # ARROW_QUERY_TIMEOUT_SECONDS +# batchSize: 1000 # ARROW_BATCH_SIZE +# # Server-side auth the sidecar enforces on incoming clients (the federation is +# # one such client). SINGLE SOURCE OF TRUTH: the chart renders BOTH the sidecar's +# # ARROW_AUTH_* env AND the federation's servers..credentials from this block. +# auth: +# method: none # none | basic | bearer | apikey +# credentialsSecretName: "" # ONE Secret feeds BOTH the sidecar's incoming +# # ARROW_AUTH_* AND the federation's outgoing +# # servers..credentials (Story 16.3, FACT A). +# # Secret keys: arrow-username/arrow-password (basic), +# # arrow-bearer-token (bearer), arrow-api-key (apikey). +# # Optional (Story 16.3): override the Secret data-key names (defaults shown). +# secretKeys: +# username: arrow-username +# password: arrow-password +# bearerToken: arrow-bearer-token # -> federation CONFIG_FORCE_..bearer__token +# apiKey: arrow-api-key # -> federation CONFIG_FORCE_..api__key +# # Optional (Story 16.3): whole-Secret envFrom mode for the SIDECAR's ARROW_AUTH_* +# # (the Secret's keys must then BE the env names, e.g. ARROW_AUTH_BEARER_TOKEN). +# # ⚠️ INCOMPATIBLE with a Secret-backed FEDERATION->sidecar credential: the federation +# # reads the same Secret per-key (arrow-bearer-token, …), which a whole-Secret-shaped +# # Secret does not carry. The chart fails fast on (method!=none + credentialsSecretName +# # + useEnvFrom=true). Use per-key mode (useEnvFrom=false) for a Secret-backed non-none +# # auth method. Default false. +# useEnvFrom: false +# # Inline creds (dev/test only) — a Secret is preferred in prod. With a Secret set, +# # the federation receives the value via CONFIG_FORCE_* (override_with_env_vars), so +# # these may be omitted (the ConfigMap then renders method-only). +# # username: "" # basic +# # password: "" # basic +# # bearerToken: "" # bearer +# # apiKey: "" # apikey +# # Per-downstream OUTGOING TLS (federation -> THIS sidecar). VERIFIED separate from +# # federation.tls (which is the federation's own INBOUND edge TLS). Maps to +# # servers..tls in the ConfigMap (DownstreamConnection forGrpcTls). In R1 the +# # arrow sidecar listens PLAINTEXT, so this is only useful for an EXTERNAL TLS Flight +# # SQL backend — intra-cluster sidecars stay plaintext. Default false. +# tls: false +# default: false # at most ONE sidecar may be default=true (bare-table routing) +# alias: "" # optional federation schema alias (defaults to `name`) +# image: # optional override of the version-derived image +# repository: "" # e.g. a custom-built sidecar +# tag: "" # defaults to .Chart.AppVersion + +# Shared defaults applied to every sidecar (each `sidecars[]` entry may override +# resources via its own `resources:` key). +sidecarDefaults: + image: + pullPolicy: IfNotPresent + resources: + requests: + memory: 768Mi + cpu: 500m + limits: + memory: 1536Mi + cpu: 1000m + # Writable /tmp for the DuckDB JNI native-lib extraction (same-cluster JOIN engine). + scratch: + sizeLimit: 1Gi + # TCP-socket probes on ARROW_PORT (the sidecar has NO gRPC health service — VERIFIED). + probes: + liveness: + initialDelaySeconds: 20 + periodSeconds: 15 + failureThreshold: 3 + readiness: + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + # Sidecars share the same hardening as the federation (overridable). + podSecurityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + seccompProfile: + type: RuntimeDefault + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + capabilities: + drop: + - ALL + +# `helm test` smoke Job (Story 16.2, hardened for CI in Story 16.5). Connects to the +# federation Flight SQL endpoint and asserts GetCatalogs (one catalog per registered +# sidecar) == len(sidecars), with a bounded connect-retry (the federation is NotReady +# until ALL downstreams are discovered — the gRPC readiness aggregate is all-or-nothing). +# The default image pip-installs ADBC at runtime (needs PyPI reachability) — pin an +# internal/pre-baked image (test.image) for air-gapped or rate-limited clusters. +test: + image: "" # default "python:3.12-slim" when empty + adbcVersion: "1.6.0" # adbc_driver_flightsql / adbc_driver_manager version (runtime pip when image not pre-baked)