diff --git a/.changeset/one-base-across-the-stack.md b/.changeset/one-base-across-the-stack.md new file mode 100644 index 00000000..dae4bc95 --- /dev/null +++ b/.changeset/one-base-across-the-stack.md @@ -0,0 +1,49 @@ +--- +"ftw": minor +--- + +Move the whole container stack to one base: Debian 13 "trixie" slim. + +Core was `alpine:3.22`, the updater sidecar was `docker:27-cli` (also alpine), +and the optimizer was `python:3.12-slim-bookworm` — two libcs, three unrelated +base images and three security streams in one deployment. Core, updater and +optimizer now all sit on `debian:trixie-slim`, verified to share a single base +layer, so a host pulls that rootfs once and there is one suite to track. It also +matches the Raspberry Pi OS release the SD image is built from. + +What the move buys: glibc, so the image can run ordinary prebuilt vendor +binaries, which musl cannot; a full userland, which makes `docker exec` +debugging on a live site practical; and `libnss-mdns` wired into +`/etc/nsswitch.conf`, so `.local` names resolve for glibc tools inside the +container — `getent hosts zap.local`, `curl`, `wget` — once an avahi socket is +mounted. Alpine has no NSS plugin mechanism at all, so none of that was +available before. + +That covers tools in the image, not the FTW process itself: the binary is built +`CGO_ENABLED=0` and so never consults NSS. It stays fully static and still +cross-compiles on the build platform. + +`wget` is now installed explicitly and asserted by the container boundary test. +It is contractual rather than incidental: `ftw-updater` `docker exec`s it inside +the core image to decide whether an update commits, and updaters already +deployed in the field will keep doing so. The zoneinfo database is also embedded +in the binary as a fallback, so a base image without `tzdata` can never silently +push `time.Local` to UTC and mis-time price and plan windows. + +Size: the core image grows from about 53 MB to about 133 MB uncompressed. The +updater is unchanged at about 203 MB — `docker:27-cli` was never a small image — +and now shares its base with the other two, so the per-host download is well +below the nominal sum. + +Data ownership is unchanged: the process still runs as the bare numeric uid 100 +/ gid 101, and gid 101 is still what grants access to the optimizer's socket. + +The optimizer is versioned independently and moves to 1.4.0, because its image +is a materially different artifact once the base changes. Its release workflow +verifies that a published image's revision label matches the commit it claims, +so the new base could not have shipped under the old version number at all. + +The base is pinned to the `trixie` codename rather than a `stable` alias so a +major-version jump can never arrive silently on a rebuild. A new scheduled +`debian base currency` workflow watches for a newer Debian stable and opens a +tracking issue when one appears. diff --git a/.github/workflows/debian-base-currency.yml b/.github/workflows/debian-base-currency.yml new file mode 100644 index 00000000..2eaaf8e0 --- /dev/null +++ b/.github/workflows/debian-base-currency.yml @@ -0,0 +1,95 @@ +name: debian base currency + +# The container images pin a Debian codename (debian:trixie-slim) rather than a +# suite alias, so that a major-version jump can never arrive silently on some +# future rebuild. The cost of pinning is that nothing notices when a new stable +# ships, and a base left behind quietly stops receiving security updates once +# its suite leaves LTS. +# +# This notices. It opens a single issue and keeps editing it, so a suite left +# alone for a year costs one thread rather than fifty-two. + +on: + schedule: + # 07:00 UTC on Mondays. Debian stable ships roughly every two years, so a + # daily check would be noise; weekly still surfaces a new release within + # days of it landing, which is far inside the window that matters. + - cron: "0 7 * * 1" + workflow_dispatch: + +permissions: + contents: read + issues: write + +concurrency: + group: debian-base-currency + cancel-in-progress: false + +jobs: + currency: + name: pinned base follows Debian stable + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Compare the pinned suite against Debian stable + id: compare + run: | + set +e + output="$(bash scripts/check-debian-base.sh 2>&1)" + status=$? + set -e + echo "$output" + { + echo "status=${status}" + echo "report<> "$GITHUB_OUTPUT" + # A network or parse failure must not read as "the base is current", + # and must not open an issue claiming a new release we never saw. + # 1 means a newer stable (or a split pin), 0 means current. + if [ "$status" != "0" ] && [ "$status" != "1" ]; then + echo "::error::the Debian currency check could not run (exit ${status})" + exit "$status" + fi + + - name: Open or update the tracking issue + if: steps.compare.outputs.status == '1' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPORT: ${{ steps.compare.outputs.report }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + title="Container base is behind Debian stable" + body="$(printf '%s\n\n```\n%s\n```\n\n%s\n\nFrom %s\n\nThis issue is rewritten by each run of the `debian base currency` workflow. Closing it without moving the pin means it comes back next Monday.\n' \ + "The Debian suite pinned by the container images is no longer the current Debian stable, or the three images have drifted apart. Core, updater and optimizer are pinned to one suite on purpose: they share a single base layer, so a host pulls that rootfs once instead of three times." \ + "${REPORT}" \ + "Moving the pin means updating the FROM lines in \`Dockerfile\`, \`Dockerfile.updater\` and \`Dockerfile.optimizer\` together, then rebuilding all three and confirming the optimizer still resolves CVXPY and HiGHS wheels on the new suite. Do not move core alone — that splits the shared layer. If the readiness lines above say a tag is not published yet, wait for it rather than splitting the pin." \ + "${RUN_URL}")" + + existing="$(gh issue list --state open --search "in:title \"${title}\"" \ + --json number,title --jq "[.[] | select(.title == \"${title}\")][0].number")" + + if [ -n "$existing" ] && [ "$existing" != "null" ]; then + gh issue edit "$existing" --body "$body" + echo "updated issue #${existing}" + else + gh issue create --title "$title" --body "$body" + fi + + - name: Close the tracking issue once the pin is current + if: steps.compare.outputs.status == '0' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + title="Container base is behind Debian stable" + existing="$(gh issue list --state open --search "in:title \"${title}\"" \ + --json number,title --jq "[.[] | select(.title == \"${title}\")][0].number")" + if [ -n "$existing" ] && [ "$existing" != "null" ]; then + gh issue close "$existing" --comment "Every image now pins the current Debian stable. Closed by the \`debian base currency\` workflow." + echo "closed issue #${existing}" + else + echo "base is current and no issue is open" + fi diff --git a/Dockerfile b/Dockerfile index 6526349c..7cc31194 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,10 @@ RUN cd go && go mod download COPY go/ ./go/ -# Cross-compile by mapping TARGETARCH → GOARCH. CGO is off so the -# binary is fully static and runs on alpine without glibc. +# Cross-compile by mapping TARGETARCH → GOARCH. CGO stays off: the binary is +# fully static, so it is the runtime's *userland* we are choosing below, not a +# libc the binary depends on. Keeping CGO off is what lets the toolchain run +# natively on the build platform instead of under emulation. ARG TARGETOS=linux ARG TARGETARCH ARG VERSION=dev @@ -36,11 +38,42 @@ RUN cd go && \ go build -trimpath -ldflags="-s -w -X main.Version=${VERSION}" \ -o /out/ftw-backup ./cmd/ftw-backup # --- Runtime --------------------------------------------------------------- -FROM alpine:3.22 +# Debian trixie-slim — current Debian stable (13), and the same suite as +# Dockerfile.updater and Dockerfile.optimizer's python:3.12-slim-trixie. One +# rootfs blob is pulled once and shared by all three images, so the extra bytes +# over alpine are paid a single time per host rather than per image, and there +# is one libc and one security stream to track. It also matches the Raspberry Pi +# OS release the SD image is built from (deploy/pi-gen/config: RELEASE=trixie). +# +# Pinned to the codename, not `stable-slim`: a suite alias would silently jump +# major versions on some future rebuild. The `debian base currency` workflow +# watches for a new stable and files an issue, so the bump stays deliberate. +# +# glibc also means the image can run ordinary prebuilt vendor binaries, which +# musl cannot, and ships a full userland for on-site debugging. +FROM debian:trixie-slim -# HTTPS integrations and timezone-aware price/plan windows need these at -# runtime. BusyBox wget provides the health check without adding Python/curl. -RUN apk add --no-cache ca-certificates tzdata +# ca-certificates — HTTPS integrations. +# tzdata — timezone-aware price/plan windows. Without a zoneinfo tree +# time.Local silently degrades to UTC and mis-times plan +# boundaries with no error, so this is load-bearing. +# wget — the HEALTHCHECK below AND ftw-updater's readiness probe, +# which `docker exec`s wget in THIS image to decide whether +# an update commits. Debian slim ships neither wget nor curl, +# so it must be installed explicitly; dropping it would make +# every self-update fail its health gate and roll back. +# libnss-mdns — resolves ".local" for glibc programs in the image (getent, +# curl, wget), so in-container debugging agrees with the +# host. apt wires mdns4_minimal into /etc/nsswitch.conf on +# install. At run time it forwards to avahi-daemon over +# /run/avahi-daemon/socket, which must be bind-mounted; see +# docs/operations.md. It does nothing for the FTW binary +# itself, which is CGO_ENABLED=0 and therefore never consults +# NSS — see the note on the builder stage above. +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates tzdata wget libnss-mdns && \ + rm -rf /var/lib/apt/lists/* # Image layout: # /app/ftw binary (immutable, replaced on upgrade) @@ -79,7 +112,13 @@ EXPOSE 8080 # with each release. # # UID note: the process runs as uid 100 / gid 101 for compatibility with -# existing bind mounts. Named docker volumes inherit ownership from the image +# existing bind mounts. These are deliberately NUMERIC — no account is created +# and none is needed, which is why ENV HOME above is load-bearing. Verified on +# this base: uid 100 and gid 101 have no passwd/group entry, so ownership simply +# renders numerically. Do not renumber: gid 101 is what grants access to the +# optimizer's 0660 socket, and existing installs (and every flashed SD card) +# already own their data dir as 100:101. +# Named docker volumes inherit ownership from the image # automatically and just work. For HOST BIND MOUNTS, the host # directory must be owned by uid 100 (or world-writable) before the # container starts: diff --git a/Dockerfile.optimizer b/Dockerfile.optimizer index f8847321..1afb7f3c 100644 --- a/Dockerfile.optimizer +++ b/Dockerfile.optimizer @@ -1,11 +1,11 @@ # Independently releasable FTW mathematical optimizer. -FROM python:3.12-slim-bookworm AS build +FROM python:3.12-slim-trixie AS build COPY optimizer/ /src/optimizer/ RUN python -m venv /opt/venv && \ /opt/venv/bin/pip install --no-cache-dir /src/optimizer -FROM python:3.12-slim-bookworm +FROM python:3.12-slim-trixie ARG VERSION=dev ARG BUILD_SHA="" diff --git a/Dockerfile.updater b/Dockerfile.updater index e00b5d5c..b002dbbe 100644 --- a/Dockerfile.updater +++ b/Dockerfile.updater @@ -28,12 +28,24 @@ RUN cd go && \ -o /out/ftw-updater ./cmd/ftw-updater # --- Runtime --------------------------------------------------------------- -# docker:27-cli bundles the `docker` binary + `docker compose` plugin + -# ca-certificates + tzdata, which is exactly what we need to execute -# compose pulls against the host daemon. -FROM docker:27-cli +# Same debian:trixie-slim rootfs as Dockerfile and Dockerfile.optimizer, so +# the whole stack pulls one base layer instead of three. docker:27-cli is +# alpine-based and was the last thing keeping a second libc in the deployment. +# +# The docker CLI and the compose plugin are copied straight out of the official +# CLI image rather than installed from Docker's apt repository: it is the same +# upstream artifact, it pins the version explicitly, and it keeps apt out of the +# emulated arm64 layer. +FROM debian:trixie-slim + +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates tzdata && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=docker:27-cli /usr/local/bin/docker /usr/local/bin/docker +COPY --from=docker:27-cli /usr/local/libexec/docker/cli-plugins/docker-compose \ + /usr/local/libexec/docker/cli-plugins/docker-compose -# Compose pull hits GHCR over TLS; docker:cli already bundles ca-certs. COPY --from=builder /out/ftw-updater /usr/local/bin/ftw-updater COPY LICENSE NOTICE /usr/share/doc/ftw/ diff --git a/Makefile b/Makefile index 8a0e1b5f..1b50629d 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,7 @@ optimizer-test: optimizer/.venv/.installed optimizer/.venv/bin/pytest -q optimizer/tests compose-migration-test: - bash -n scripts/enable-modular-stack.sh scripts/migrate-legacy-compose.sh scripts/install-macos.sh scripts/deploy-home-link-web.sh scripts/sync-bundled-drivers.sh scripts/check-driver-versions.sh + bash -n scripts/enable-modular-stack.sh scripts/migrate-legacy-compose.sh scripts/install-macos.sh scripts/deploy-home-link-web.sh scripts/sync-bundled-drivers.sh scripts/check-driver-versions.sh scripts/check-debian-base.sh bash scripts/test-modular-compose.sh container-boundary-test: diff --git a/deploy/pi-gen/stage-ftw/01-ftw-setup/00-run.sh b/deploy/pi-gen/stage-ftw/01-ftw-setup/00-run.sh index 9e8aaa23..581a86d8 100755 --- a/deploy/pi-gen/stage-ftw/01-ftw-setup/00-run.sh +++ b/deploy/pi-gen/stage-ftw/01-ftw-setup/00-run.sh @@ -69,8 +69,9 @@ EOF # records /opt/ftw automatically. COMPOSE_PROJECT_NAME # stays the literal `ftw`. # -# data/ is owned 100:101 because the in-container ftw user (alpine -# `adduser -S`) needs to own it before SQLite can create state.db. +# data/ is owned 100:101 because the container process runs as that bare +# numeric uid/gid (no account exists in the image) and needs to own the +# directory before SQLite can create state.db. # Same UID/GID mapping as scripts/install.sh. install -d -m 0755 "${ROOTFS_DIR}/opt/ftw" install -d -m 0755 -o 100 -g 101 "${ROOTFS_DIR}/opt/ftw/data" diff --git a/docs/operations.md b/docs/operations.md index 8e95e98a..b1a97cee 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -180,6 +180,31 @@ Verify the broker address from the same network namespace as core, then inspect broker and driver logs. Device credentials and topic mappings belong to the driver configuration. +### `.local` names inside the container + +The image ships `libnss-mdns`, so ordinary glibc tools inside the container — +`getent hosts zap.local`, `curl`, `wget` — resolve `.local` the way they do on +the host. `apt` wires `mdns4_minimal [NOTFOUND=return]` into +`/etc/nsswitch.conf` when the package is installed; nothing else is needed at +build time. + +At run time that path talks to `avahi-daemon` over a Unix socket, and a socket +is not shared by host networking the way a port is. Mount it explicitly: + +```yaml + volumes: + - /run/avahi-daemon/socket:/run/avahi-daemon/socket:ro +``` + +Only add this on a host that actually runs `avahi-daemon` — the Raspberry Pi +image does. Without the daemon Docker creates a *directory* at that path, which +resolves nothing and is harmless but confusing; `ls -l` there is the quickest +way to tell the two apart. + +This makes the container's own tooling agree with the host. Whether the FTW +process itself resolves a device's `.local` name is a separate question, +answered by `internal/mdnsresolve`. + ### Configuration rejected Read the validation error, compare with diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 509f7946..de9f05ca 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -25,6 +25,14 @@ import ( "syscall" "time" + // Embed the zoneinfo database as a fallback. The image's system tree still + // wins whenever it is present; this exists so time.Local can never silently + // degrade to UTC and mis-time price and plan windows if a base image ships + // without tzdata. The failure it guards against is invisible — production + // code reads time.Local, and the tzdata tests skip rather than fail — so the + // ~450 KB is worth it. + _ "time/tzdata" + "github.com/srcfl/ftw/go/internal/api" "github.com/srcfl/ftw/go/internal/arp" "github.com/srcfl/ftw/go/internal/battery" diff --git a/optimizer/pyproject.toml b/optimizer/pyproject.toml index 2611fca7..8ef2a8de 100644 --- a/optimizer/pyproject.toml +++ b/optimizer/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "ftw-optimizer" -version = "1.3.2" +version = "1.4.0" description = "CVXPY planning engine for ftw" requires-python = ">=3.11" dependencies = [ diff --git a/scripts/check-debian-base.sh b/scripts/check-debian-base.sh new file mode 100755 index 00000000..9fb7b963 --- /dev/null +++ b/scripts/check-debian-base.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Reports whether the Debian suite pinned by the container images is still the +# current Debian stable. +# +# The images pin a codename (debian:trixie-slim) rather than a suite alias +# (debian:stable-slim) so a major-version jump can never arrive silently on a +# rebuild. The cost of pinning is that nothing notices when a new stable ships — +# this is what notices. +# +# Truth comes from Debian's own Release file for the `stable` suite, not from +# registry tag listings: tags are noisy, rate-limited and say nothing about +# which suite Debian considers stable. +# +# Exit codes, matching scripts/sync-bundled-drivers.sh --behind: +# 0 pinned suite is current stable +# 1 a newer stable exists, or the Dockerfiles disagree with each other +# 2 the check could not run (network, parse) — never reported as "fine" +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "$ROOT" + +RELEASE_URL="${DEBIAN_RELEASE_URL:-https://deb.debian.org/debian/dists/stable/Release}" + +# Read the pin out of the Dockerfiles instead of hard-coding it here, so this +# check cannot drift away from what actually ships. +pinned_debian() { sed -n 's/^FROM debian:\([a-z][a-z]*\)-slim.*/\1/p' "$1" | head -1; } +pinned_python() { sed -n 's/^FROM python:[0-9.][0-9.]*-slim-\([a-z][a-z]*\).*/\1/p' "$1" | head -1; } + +core=$(pinned_debian Dockerfile) +updater=$(pinned_debian Dockerfile.updater) +optimizer=$(pinned_python Dockerfile.optimizer) + +for pair in "Dockerfile:$core" "Dockerfile.updater:$updater" "Dockerfile.optimizer:$optimizer"; do + if [ -z "${pair#*:}" ]; then + echo "could not read a Debian suite from ${pair%%:*}" >&2 + exit 2 + fi +done + +echo "pinned suite:" +printf ' %-22s %s\n' "Dockerfile" "$core" "Dockerfile.updater" "$updater" "Dockerfile.optimizer" "$optimizer" + +if [ "$core" != "$updater" ] || [ "$core" != "$optimizer" ]; then + echo "" + echo "The three images no longer agree on one Debian suite. Sharing a single" + echo "base layer is the reason they were aligned, and that benefit is lost" + echo "while they differ." + exit 1 +fi + +release=$(curl -fsSL --max-time 20 "$RELEASE_URL" 2>/dev/null) || { + echo "could not fetch $RELEASE_URL" >&2 + exit 2 +} +stable=$(printf '%s\n' "$release" | sed -n 's/^Codename: *//p' | head -1) +version=$(printf '%s\n' "$release" | sed -n 's/^Version: *//p' | head -1) +if [ -z "$stable" ]; then + echo "no Codename field in $RELEASE_URL" >&2 + exit 2 +fi + +echo "" +echo "debian stable: $stable${version:+ (}${version}${version:+)}" + +if [ "$core" = "$stable" ]; then + echo "" + echo "The pinned suite is current." + exit 0 +fi + +echo "" +echo "A newer Debian stable is available: $core -> $stable" + +# Advisory only. A new Debian stable is tagged in the official images promptly, +# but python:-slim- can lag by days, and moving core without the +# optimizer would split the shared base layer. Never fail the check on this — +# it is a readiness note, not the finding. +if command -v docker >/dev/null 2>&1; then + echo "" + echo "image readiness:" + python_tag=$(sed -n 's/^FROM \(python:[0-9.][0-9.]*\)-slim-[a-z][a-z]*.*/\1/p' Dockerfile.optimizer | head -1) + for image in "debian:${stable}-slim" "${python_tag}-slim-${stable}"; do + if docker manifest inspect "$image" >/dev/null 2>&1; then + printf ' %-32s available\n' "$image" + else + printf ' %-32s NOT PUBLISHED YET\n' "$image" + fi + done +fi + +exit 1 diff --git a/scripts/install.sh b/scripts/install.sh index 33179091..942f28a5 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -149,8 +149,8 @@ fi echo "" echo "==[3/5]== Preparing install directory: $INSTALL_DIR" mkdir -p "$INSTALL_DIR/data" -# The image runs as uid 100 / gid 101 (the `ftw` user created in the -# alpine runtime stage — see Dockerfile). A bind-mounted host dir must +# The image runs as uid 100 / gid 101 — a bare numeric USER, no account is +# created in the image at all (see Dockerfile). A bind-mounted host dir must # match those IDs so SQLite can create state.db inside it. $SUDO chown -R 100:101 "$INSTALL_DIR/data" diff --git a/scripts/test-container-boundaries.sh b/scripts/test-container-boundaries.sh index ea827f6b..1b4c40d5 100755 --- a/scripts/test-container-boundaries.sh +++ b/scripts/test-container-boundaries.sh @@ -9,7 +9,24 @@ if grep -Eq 'COPY optimizer/|--from=optimizer|/opt/venv|FTW_OPTIMIZER_(PYTHON|DI exit 1 fi -grep -q '^FROM alpine:' Dockerfile +# The core runtime must stay a small, Python-free userland. This used to be +# asserted as `^FROM alpine:`, which was a proxy for the real rule and gave no +# diagnostic when it failed — Python originally reached core precisely by way of +# the optimizer's base image, so the check existed to catch a base that drags an +# interpreter in, not to mandate one distro. Assert that directly instead. +if grep -Eq '^FROM .*(python|pypy)' Dockerfile; then + echo "Dockerfile must not build on a Python base image; use Dockerfile.optimizer for Python/CVXPY" >&2 + exit 1 +fi +# wget is contractual, not incidental: the HEALTHCHECK uses it, and +# ftw-updater docker-execs it inside this image to decide whether an update +# commits. An updater already deployed in the field will keep doing so, so the +# core image cannot stop shipping wget without breaking self-update on every +# host running an older sidecar. +if ! grep -Eq 'wget' Dockerfile; then + echo "Dockerfile must provide wget: the HEALTHCHECK and ftw-updater's readiness probe both exec it" >&2 + exit 1 +fi grep -q '^COPY optimizer/' Dockerfile.optimizer grep -q '/out/ftw-backup' Dockerfile grep -q '/app/ftw-backup' Dockerfile