diff --git a/.buildkite/README.md b/.buildkite/README.md new file mode 100644 index 0000000..ddb407b --- /dev/null +++ b/.buildkite/README.md @@ -0,0 +1,172 @@ +# AKernel Buildkite Image Pipeline + +This pipeline builds and pushes one universal AKernel image. The same image +runs as master, frontend, node, or standalone; the deployment target setting +controls only which deployment bundles are uploaded. + +## Source selection + +The default source is the checksum-published YuanRong release: + +```text +YR_SOURCE=release +YR_VERSION=0.9.7 +``` + +To consume an existing passed YuanRong Buildkite build: + +```text +YR_SOURCE=buildkite +YR_BUILDKITE_ORG=openyuanrong +YR_PIPELINE=yuanrong-jcl +YR_BUILD_NUMBER=221 +``` + +The resolver reads the build's `obs-urls.*` metadata and does not scrape job +logs. The encrypted Buildkite Secret `YR_BUILDKITE_API_TOKEN`, restricted to +the `akernel-image` pipeline, is injected only into the resolver job. The +resolver accepts these read-capable environment variables in priority order: + +1. `YR_BUILDKITE_API_TOKEN` +2. `BUILDKITE_API_TOKEN` +3. `BUILDKITE_PACKAGE_UPLOAD_TOKEN` + +Do not pass an API token as a build environment override. Store it in the +pipeline-scoped encrypted Secret instead; image-build and packaging jobs do +not receive it. + +## Image and deployment inputs + +| Variable | Default | Values | +| --- | --- | --- | +| `AKERNEL_IMAGE_REPOSITORY` | `swr.cn-southwest-2.myhuaweicloud.com/openyuanrong/akernel-all-in-one` | Docker repository | +| `AKERNEL_IMAGE_TAG` | generated | valid Docker tag | +| `AKERNEL_DEPLOY_TARGETS` | `standalone,helm` | `standalone`, `helm`, or both | +| `AKERNEL_INCLUDE_KATA` | `true` | `true` or `false` | +| `AKERNEL_INCLUDE_NVIDIA` | `true` | `true` or `false` | +| `AKERNEL_BUILDKITE_BUILDER_IMAGE` | YuanRong sandbox packager | privileged builder image | + +Generated tags contain the normalized AKernel branch, AKernel Buildkite build +number, short AKernel commit, and YuanRong release version or source build +number. Deployment bundles record the registry digest returned after push. + +The image job uses the existing Buildkite Kubernetes `default` queue. Its pod +receives `SWR_USERNAME`, `SWR_PASSWORD`, and Docker configuration from the +existing `swr-credentials` and `swr-pull-secret` secrets. No registry secret is +stored in this repository or uploaded as an artifact. + +## Large dependency cache + +Create `/mnt/paas/build-cache/akernel-dependency-cache` on the selected amd64 +Buildkite node, then apply +`.buildkite/kubernetes/akernel-dependency-cache-pvc.yaml` once in the Buildkite +job namespace before running the pipeline. The manifest defines a static +10 GiB local PV on node `10.10.189.4`, a `kubernetes.io/no-provisioner` +StorageClass with `WaitForFirstConsumer`, and the `ReadWriteOnce` claim. The +volume and class both use `Retain`, so deleting the claim cannot delete the +host cache directory. If the cache node is replaced, update the PV node +affinity and create the directory on its replacement before recreating the PV. + +The claim is mounted read-write at `/var/cache/akernel-downloads` only in the +image command container; checkout, YuanRong resolution, and deployment +packaging do not mount it. `/var/lib/docker` remains a per-job `emptyDir` and +is never shared between Docker daemons. + +The cache inventory is deliberately limited to these checksum-pinned, +immutable download artifacts. Cache paths include component, version, +architecture, digest, and filename: + +| Component | Cache path | Digest | Size | +| --- | --- | --- | --- | +| Kata Containers 4.0.0 amd64 static archive | `kata/4.0.0/amd64/2c3b9dfeba355582b40aee462b12916c9740654d0230f696adf719d67b063a8c/kata-static-4.0.0-amd64.tar.zst` | SHA-256 `2c3b9dfeba355582b40aee462b12916c9740654d0230f696adf719d67b063a8c` | 1,952,994,060 bytes | +| gVisor `runsc` `release-20260706.0` x86_64 binary | `gvisor/release-20260706.0/x86_64/73938c145ebe554cf61a01da455688f4b732eebdf7b1b635bdef5b195868b363d8cb400e3d92ed1f377b78996805556c247a4849583910cb04e92b156053033e/runsc` | SHA-512 `73938c145ebe554cf61a01da455688f4b732eebdf7b1b635bdef5b195868b363d8cb400e3d92ed1f377b78996805556c247a4849583910cb04e92b156053033e` | 130,918,823 bytes | +| OpenTelemetry Collector contrib 0.120.0 linux amd64 archive | `otelcol-contrib/0.120.0/linux-amd64/81bf885bc9a86705feb3c113c5a356571390e3601eb651ffcf2b3428f6571adb/otelcol-contrib_0.120.0_linux_amd64.tar.gz` | SHA-256 `81bf885bc9a86705feb3c113c5a356571390e3601eb651ffcf2b3428f6571adb` | 80,901,637 bytes | + +The three entries total 2,164,814,520 bytes, so the 10 GiB PV remains +sufficient with 8,572,603,720 bytes available before filesystem overhead. +Do not expand the PVC to Docker, Cargo, Go, apt, source, or other caches. + +The host downloader verifies a hit before reporting it. A miss, or a corrupt +existing entry, is downloaded to a build-unique temporary file on the PVC, +verified with its pinned SHA-256 or SHA-512 digest, and atomically replaces the +destination. Docker mounts the cache read-only and independently verifies the +same digest before installing each artifact. Cache entries are disposable; +deleting or corrupting one only makes the next image job safely replace it. + +`GVISOR_RELEASE` and `GVISOR_AMD64_SHA512` are a required override pair, as +are `OTELCOL_CONTRIB_VERSION` and `OTELCOL_CONTRIB_SHA256`: provide both +values together or neither. `GVISOR_RELEASE_BASE_URL` and an exact +`OTELCOL_CONTRIB_URL` may select an HTTP(S) mirror without changing the +version/digest pairing. The gVisor binary remains installed at +`/usr/local/bin/runsc`; the OpenTelemetry configuration and systemd wiring are +unchanged. + +Outside Buildkite, leave `AKERNEL_DEPENDENCY_CACHE_DIR` unset to retain the +normal direct-download fallback for Kata, gVisor, and OpenTelemetry. Set it to +a writable directory to opt into the same verified cache behavior. + +## Restricted GitHub egress + +Every job requires the encrypted Buildkite Secret `AKERNEL_WG_CONFIG`. Its +policy must allow only the `akernel-image` pipeline. The Kubernetes PodSpec +installs an idempotent environment hook before checkout: it brings up `wg0` +only when the interface is absent and exports the HTTP(S) proxy at +`10.77.0.1:3128`. The WireGuard private key remains in Buildkite Secrets and is +never stored in this repository, pipeline YAML, metadata, logs, or artifacts. + +The hook leaves Buildkite, its Amazon S3 artifact store, Kubernetes/private +networks, and Huawei Cloud/SWR domains in `NO_PROXY`; only the proxy server's +`10.77.0.1/32` address is routed through WireGuard. Both checkout and command +containers receive +`BUILDKITE_HOOKS_PATH` through the Kubernetes PodSpec because Buildkite treats +that variable as protected. Automatic recursive submodule checkout remains +disabled; the image job initializes only `src/sandboxd` and `src/distill-fs`. + +The PodSpec also supplies `AKERNEL_WG_ENDPOINT_OVERRIDE=159.138.22.93:443`. +Before starting WireGuard, the hook rewrites only the `Endpoint` entry in the +temporary client configuration; it neither reads nor replaces the encrypted +client private key. The Hong Kong security group must allow UDP 443 only from +the Guiyang Buildkite egress address. Squid remains bound exclusively to +`10.77.0.1:3128` and is never exposed as a public proxy. + +The current Alpine-based containers install `wireguard-tools`, `iproute2`, +`curl`, and `git` from the Alibaba Cloud mirror when needed. Replace them with +a prebuilt checkout image only if measured job startup time justifies it. + +## Outputs + +Every successful build uploads: + +- `openyuanrong_sandbox-*.whl` +- `artifact-manifest.json` +- `image-manifest.json` +- `SHA256SUMS` + +It also uploads the requested deployment products: + +- `akernel-standalone-.tar.gz` +- `akernel-helm-.tgz` + +The standalone bundle contains `deploy/standalone`, `image.env`, the sandbox +SDK, and both manifests. It excludes local `deploy/standalone/data` state. +The Helm bundle contains `deploy/akernel`, a generated `values.image.yaml`, the +sandbox SDK, and both manifests. + +## Local validation + +Install PyYAML for the pipeline parser tests, then run: + +```bash +python3 -m pip install 'PyYAML>=6,<7' +make buildkite-check +``` + +On macOS systems without `/usr/bin/bash`, use: + +```bash +make SHELL=/bin/bash buildkite-check +``` + +The gate runs resolver fixtures over a real local HTTP server, image-wrapper +tests with command-boundary fakes, real archive inspection, dynamic YAML +parsing, Python compilation, and Bash syntax checks. diff --git a/.buildkite/kubernetes/akernel-dependency-cache-pvc.yaml b/.buildkite/kubernetes/akernel-dependency-cache-pvc.yaml new file mode 100644 index 0000000..d9f35e8 --- /dev/null +++ b/.buildkite/kubernetes/akernel-dependency-cache-pvc.yaml @@ -0,0 +1,42 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: akernel-local-cache +provisioner: kubernetes.io/no-provisioner +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: akernel-dependency-cache-pv +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: akernel-local-cache + local: + path: /mnt/paas/build-cache/akernel-dependency-cache + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - 10.10.189.4 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: akernel-dependency-cache +spec: + storageClassName: akernel-local-cache + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/.buildkite/pipeline.sh b/.buildkite/pipeline.sh new file mode 100755 index 0000000..3ac6238 --- /dev/null +++ b/.buildkite/pipeline.sh @@ -0,0 +1,327 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +yr_source="${YR_SOURCE:-release}" +yr_version="${YR_VERSION:-0.9.7}" +yr_org="${YR_BUILDKITE_ORG:-openyuanrong}" +yr_pipeline="${YR_PIPELINE:-yuanrong-jcl}" +yr_build_number="${YR_BUILD_NUMBER:-}" +deploy_targets="${AKERNEL_DEPLOY_TARGETS:-standalone,helm}" +image_repository="${AKERNEL_IMAGE_REPOSITORY:-swr.cn-southwest-2.myhuaweicloud.com/openyuanrong/akernel-all-in-one}" +image_tag="${AKERNEL_IMAGE_TAG:-}" +include_kata="${AKERNEL_INCLUDE_KATA:-true}" +include_nvidia="${AKERNEL_INCLUDE_NVIDIA:-true}" +builder_image="${AKERNEL_BUILDKITE_BUILDER_IMAGE:-swr.cn-southwest-2.myhuaweicloud.com/yuanrong-dev/sandbox-packager:v20260506_kubectl}" +pip_index_url="${PIP_INDEX_URL:-https://mirrors.huaweicloud.com/repository/pypi/simple}" +uv_python_install_mirror="${UV_PYTHON_INSTALL_MIRROR:-}" +wg_endpoint_override="${AKERNEL_WG_ENDPOINT_OVERRIDE:-159.138.22.93:443}" +egress_hook_base64="IyEvYmluL3NoCnNldCAtZXUKaWYgISBjb21tYW5kIC12IHdnID4vZGV2L251bGwgMj4mMTsgdGhlbgogIHNlZCAtaSAncy9kbC1jZG4uYWxwaW5lbGludXgub3JnL21pcnJvcnMuYWxpeXVuLmNvbS9nJyAvZXRjL2Fway9yZXBvc2l0b3JpZXMKICBhcGsgYWRkIC0tbm8tY2FjaGUgd2lyZWd1YXJkLXRvb2xzIGlwcm91dGUyIGN1cmwgZ2l0IG1ha2UgPi9kZXYvbnVsbApmaQppZiAhIGlwIGxpbmsgc2hvdyB3ZzAgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgdW1hc2sgMDc3CiAgcHJpbnRmICclc1xuJyAiJEFLRVJORUxfV0dfQ09ORklHIiA+IC90bXAvd2cwLmNvbmYKICBpZiBbIC1uICIke0FLRVJORUxfV0dfRU5EUE9JTlRfT1ZFUlJJREU6LX0iIF07IHRoZW4KICAgIGF3ayAtdiBlbmRwb2ludD0iJEFLRVJORUxfV0dfRU5EUE9JTlRfT1ZFUlJJREUiICcKICAgICAgQkVHSU4geyB1cGRhdGVkID0gMCB9CiAgICAgIC9eW1s6c3BhY2U6XV0qRW5kcG9pbnRbWzpzcGFjZTpdXSo9LyB7CiAgICAgICAgcHJpbnQgIkVuZHBvaW50ID0gIiBlbmRwb2ludAogICAgICAgIHVwZGF0ZWQgPSAxCiAgICAgICAgbmV4dAogICAgICB9CiAgICAgIHsgcHJpbnQgfQogICAgICBFTkQgeyBpZiAoIXVwZGF0ZWQpIGV4aXQgMSB9CiAgICAnIC90bXAvd2cwLmNvbmYgPiAvdG1wL3dnMC5jb25mLm92ZXJyaWRlCiAgICBtdiAvdG1wL3dnMC5jb25mLm92ZXJyaWRlIC90bXAvd2cwLmNvbmYKICBmaQogIHdnLXF1aWNrIHVwIC90bXAvd2cwLmNvbmYKZmkKZXhwb3J0IEhUVFBfUFJPWFk9aHR0cDovLzEwLjc3LjAuMTozMTI4CmV4cG9ydCBIVFRQU19QUk9YWT1odHRwOi8vMTAuNzcuMC4xOjMxMjgKZXhwb3J0IGh0dHBfcHJveHk9aHR0cDovLzEwLjc3LjAuMTozMTI4CmV4cG9ydCBodHRwc19wcm94eT1odHRwOi8vMTAuNzcuMC4xOjMxMjgKZXhwb3J0IE5PX1BST1hZPTEyNy4wLjAuMSxsb2NhbGhvc3QsLnN2YywuY2x1c3Rlci5sb2NhbCwxMC4wLjAuMC84LDEwMC42NC4wLjAvMTAsMTcyLjE2LjAuMC8xMiwxOTIuMTY4LjAuMC8xNiwubXlodWF3ZWljbG91ZC5jb20sLmh1YXdlaWNsb3VkLmNvbSwuYnVpbGRraXRlLmNvbSxidWlsZGtpdGVhcnRpZmFjdHMuY29tLC5idWlsZGtpdGVhcnRpZmFjdHMuY29tLC5hbWF6b25hd3MuY29tCmV4cG9ydCBub19wcm94eT0iJE5PX1BST1hZIgo=" + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +case "${yr_source}" in + release|buildkite) ;; + *) die "YR_SOURCE must be release or buildkite" ;; +esac +[[ "${yr_version}" =~ ^[0-9A-Za-z][0-9A-Za-z.+_-]*$ ]] || \ + die "YR_VERSION is invalid" +[[ "${yr_org}" =~ ^[0-9A-Za-z][0-9A-Za-z_-]*$ ]] || \ + die "YR_BUILDKITE_ORG is invalid" +[[ "${yr_pipeline}" =~ ^[0-9A-Za-z][0-9A-Za-z_-]*$ ]] || \ + die "YR_PIPELINE is invalid" +if [[ "${yr_source}" == "buildkite" ]]; then + [[ "${yr_build_number}" =~ ^[1-9][0-9]*$ ]] || \ + die "YR_BUILD_NUMBER must be a positive integer for Buildkite sources" +fi +[[ "${image_repository}" =~ ^[A-Za-z0-9._:-]+(/[A-Za-z0-9._-]+)+$ ]] || \ + die "AKERNEL_IMAGE_REPOSITORY is invalid" +if [[ -n "${image_tag}" ]]; then + [[ "${image_tag}" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]] || \ + die "AKERNEL_IMAGE_TAG is invalid" +fi +case "${include_kata}" in true|false) ;; *) die "AKERNEL_INCLUDE_KATA must be true or false" ;; esac +case "${include_nvidia}" in true|false) ;; *) die "AKERNEL_INCLUDE_NVIDIA must be true or false" ;; esac +[[ "${builder_image}" =~ ^[A-Za-z0-9._:/@-]+$ ]] || \ + die "AKERNEL_BUILDKITE_BUILDER_IMAGE is invalid" +[[ "${wg_endpoint_override}" =~ ^[A-Za-z0-9.:-]+$ ]] || \ + die "AKERNEL_WG_ENDPOINT_OVERRIDE is invalid" +[[ "${pip_index_url}" =~ ^https?://[^[:space:]\"]+$ ]] || \ + die "PIP_INDEX_URL is invalid" +if [[ -n "${uv_python_install_mirror}" ]]; then + [[ "${uv_python_install_mirror}" =~ ^https?://[^[:space:]\"]+$ ]] || \ + die "UV_PYTHON_INSTALL_MIRROR is invalid" +fi + +normalized_targets="" +IFS=',' read -r -a requested_targets <<<"${deploy_targets}" +for requested_target in "${requested_targets[@]}"; do + target="$(printf '%s' "${requested_target}" | tr -d '[:space:]')" + case "${target}" in + standalone|helm) ;; + *) die "unsupported deployment target: ${target:-empty}" ;; + esac + case ",${normalized_targets}," in + *",${target},"*) ;; + *) normalized_targets="${normalized_targets:+${normalized_targets},}${target}" ;; + esac +done +[[ -n "${normalized_targets}" ]] || die "at least one deployment target is required" + +cat < /hooks/environment; chmod 755 /hooks/environment" + volumeMounts: + - name: agent-hooks + mountPath: /hooks + containers: + - name: checkout + env: + - name: BUILDKITE_HOOKS_PATH + value: /buildkite/hooks + - name: AKERNEL_WG_ENDPOINT_OVERRIDE + value: "${wg_endpoint_override}" + securityContext: + capabilities: + add: ["NET_ADMIN"] + - name: container-0 + image: "${builder_image}" + env: + - name: BUILDKITE_HOOKS_PATH + value: /buildkite/hooks + - name: AKERNEL_WG_ENDPOINT_OVERRIDE + value: "${wg_endpoint_override}" + securityContext: + capabilities: + add: ["NET_ADMIN"] + resources: + requests: { cpu: "1", memory: "1Gi" } + limits: { cpu: "2", memory: "2Gi" } + volumes: + - name: agent-hooks + emptyDir: {} + timeout_in_minutes: 20 + + - label: ":docker: Build and push universal AKernel image" + key: "build-image" + depends_on: "resolve-yuanrong" + command: | + set -euo pipefail + wg show wg0 + rm -rf artifacts/yuanrong artifacts/image + buildkite-agent artifact download "artifacts/yuanrong/*" . --step resolve-yuanrong + mkdir -p artifacts/image + trap 'buildkite-agent artifact upload "artifacts/image/*" || true' EXIT + bash .buildkite/scripts/build_and_push.sh + secrets: + - AKERNEL_WG_CONFIG + env: + YR_ARTIFACT_MANIFEST: "artifacts/yuanrong/artifact-manifest.json" + AKERNEL_IMAGE_MANIFEST: "artifacts/image/image-manifest.json" + AKERNEL_IMAGE_REPOSITORY: "${image_repository}" + AKERNEL_IMAGE_TAG: "${image_tag}" + AKERNEL_INCLUDE_KATA: "${include_kata}" + AKERNEL_INCLUDE_NVIDIA: "${include_nvidia}" + AKERNEL_DEPENDENCY_CACHE_DIR: "/var/cache/akernel-downloads" + PIP_INDEX_URL: "${pip_index_url}" + UV_PYTHON_INSTALL_MIRROR: "${uv_python_install_mirror}" + agents: + queue: "default" + os: "linux" + arch: "amd64" + plugins: + - kubernetes: + extraVolumeMounts: + - name: agent-hooks + mountPath: /buildkite/hooks + podSpecPatch: + imagePullSecrets: + - name: swr-pull-secret + initContainers: + - name: install-egress-hook + image: "${builder_image}" + command: + - /bin/sh + - -ec + - "printf '%s' '${egress_hook_base64}' | base64 -d > /hooks/environment; chmod 755 /hooks/environment" + volumeMounts: + - name: agent-hooks + mountPath: /hooks + containers: + - name: checkout + env: + - name: BUILDKITE_HOOKS_PATH + value: /buildkite/hooks + - name: AKERNEL_WG_ENDPOINT_OVERRIDE + value: "${wg_endpoint_override}" + securityContext: + capabilities: + add: ["NET_ADMIN"] + - name: container-0 + image: "${builder_image}" + securityContext: + privileged: true + capabilities: + add: ["NET_ADMIN"] + volumeMounts: + - name: docker-graph + mountPath: /var/lib/docker + - name: dependency-cache + mountPath: /var/cache/akernel-downloads + resources: + requests: { cpu: "8", memory: "16Gi" } + limits: { cpu: "10", memory: "32Gi" } + env: + - name: BUILDKITE_HOOKS_PATH + value: /buildkite/hooks + - name: AKERNEL_WG_ENDPOINT_OVERRIDE + value: "${wg_endpoint_override}" + - name: SWR_USERNAME + valueFrom: + secretKeyRef: { name: swr-credentials, key: username, optional: true } + - name: SWR_PASSWORD + valueFrom: + secretKeyRef: { name: swr-credentials, key: password, optional: true } + - name: SWR_DOCKER_CONFIG_JSON + valueFrom: + secretKeyRef: { name: swr-pull-secret, key: .dockerconfigjson, optional: true } + volumes: + - name: agent-hooks + emptyDir: {} + - name: docker-graph + emptyDir: + sizeLimit: 100Gi + - name: dependency-cache + persistentVolumeClaim: + claimName: akernel-dependency-cache + timeout_in_minutes: 180 + + - label: ":package: Package AKernel deployments" + key: "package-deployments" + depends_on: "build-image" + command: | + set -euo pipefail + wg show wg0 + rm -rf artifacts/yuanrong artifacts/image artifacts/packages + buildkite-agent artifact download "artifacts/yuanrong/*" . --step resolve-yuanrong + buildkite-agent artifact download "artifacts/image/image-manifest.json" . --step build-image + set -- artifacts/yuanrong/openyuanrong_sandbox-*.whl + [ "\$\$#" -eq 1 ] || { echo "expected exactly one sandbox SDK wheel" >&2; exit 1; } + python3 .buildkite/scripts/package_deployments.py \\ + --repo-root . \\ + --artifact-manifest artifacts/yuanrong/artifact-manifest.json \\ + --image-manifest artifacts/image/image-manifest.json \\ + --sandbox-sdk "\$\$1" \\ + --output-dir artifacts/packages \\ + --targets "\$\$AKERNEL_DEPLOY_TARGETS" + buildkite-agent artifact upload "artifacts/packages/*" + secrets: + - AKERNEL_WG_CONFIG + env: + AKERNEL_DEPLOY_TARGETS: "${normalized_targets}" + agents: + queue: "default" + os: "linux" + arch: "amd64" + plugins: + - kubernetes: + extraVolumeMounts: + - name: agent-hooks + mountPath: /buildkite/hooks + podSpecPatch: + imagePullSecrets: + - name: swr-pull-secret + initContainers: + - name: install-egress-hook + image: "${builder_image}" + command: + - /bin/sh + - -ec + - "printf '%s' '${egress_hook_base64}' | base64 -d > /hooks/environment; chmod 755 /hooks/environment" + volumeMounts: + - name: agent-hooks + mountPath: /hooks + containers: + - name: checkout + env: + - name: BUILDKITE_HOOKS_PATH + value: /buildkite/hooks + - name: AKERNEL_WG_ENDPOINT_OVERRIDE + value: "${wg_endpoint_override}" + securityContext: + capabilities: + add: ["NET_ADMIN"] + - name: container-0 + image: "${builder_image}" + env: + - name: BUILDKITE_HOOKS_PATH + value: /buildkite/hooks + - name: AKERNEL_WG_ENDPOINT_OVERRIDE + value: "${wg_endpoint_override}" + securityContext: + capabilities: + add: ["NET_ADMIN"] + resources: + requests: { cpu: "1", memory: "2Gi" } + limits: { cpu: "2", memory: "4Gi" } + volumes: + - name: agent-hooks + emptyDir: {} + timeout_in_minutes: 20 +YAML diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000..55e8f41 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,67 @@ +# Buildkite bootstrap for the repository-owned dynamic pipeline. + +checkout: + submodules: false + +steps: + - label: ":pipeline: Initialize AKernel image build" + key: "initialize" + command: | + set -euo pipefail + wg show wg0 + curl -fsSIL --max-time 30 https://github.com/ -o /dev/null + denied=$$(curl -sS -o /dev/null -w '%{http_connect}' --max-time 20 https://example.com/ || true) + test "$$denied" = "403" + echo "restricted-egress-ok non-whitelist=$$denied" + bash .buildkite/pipeline.sh | buildkite-agent pipeline upload + secrets: + - AKERNEL_WG_CONFIG + agents: + queue: "default" + os: "linux" + arch: "amd64" + plugins: + - kubernetes: + extraVolumeMounts: + - name: agent-hooks + mountPath: /buildkite/hooks + podSpecPatch: + imagePullSecrets: + - name: swr-pull-secret + initContainers: + - name: install-egress-hook + image: "swr.cn-southwest-2.myhuaweicloud.com/yuanrong-dev/sandbox-packager:v20260506_kubectl" + command: + - /bin/sh + - -ec + - "printf '%s' 'IyEvYmluL3NoCnNldCAtZXUKaWYgISBjb21tYW5kIC12IHdnID4vZGV2L251bGwgMj4mMTsgdGhlbgogIHNlZCAtaSAncy9kbC1jZG4uYWxwaW5lbGludXgub3JnL21pcnJvcnMuYWxpeXVuLmNvbS9nJyAvZXRjL2Fway9yZXBvc2l0b3JpZXMKICBhcGsgYWRkIC0tbm8tY2FjaGUgd2lyZWd1YXJkLXRvb2xzIGlwcm91dGUyIGN1cmwgZ2l0IG1ha2UgPi9kZXYvbnVsbApmaQppZiAhIGlwIGxpbmsgc2hvdyB3ZzAgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgdW1hc2sgMDc3CiAgcHJpbnRmICclc1xuJyAiJEFLRVJORUxfV0dfQ09ORklHIiA+IC90bXAvd2cwLmNvbmYKICBpZiBbIC1uICIke0FLRVJORUxfV0dfRU5EUE9JTlRfT1ZFUlJJREU6LX0iIF07IHRoZW4KICAgIGF3ayAtdiBlbmRwb2ludD0iJEFLRVJORUxfV0dfRU5EUE9JTlRfT1ZFUlJJREUiICcKICAgICAgQkVHSU4geyB1cGRhdGVkID0gMCB9CiAgICAgIC9eW1s6c3BhY2U6XV0qRW5kcG9pbnRbWzpzcGFjZTpdXSo9LyB7CiAgICAgICAgcHJpbnQgIkVuZHBvaW50ID0gIiBlbmRwb2ludAogICAgICAgIHVwZGF0ZWQgPSAxCiAgICAgICAgbmV4dAogICAgICB9CiAgICAgIHsgcHJpbnQgfQogICAgICBFTkQgeyBpZiAoIXVwZGF0ZWQpIGV4aXQgMSB9CiAgICAnIC90bXAvd2cwLmNvbmYgPiAvdG1wL3dnMC5jb25mLm92ZXJyaWRlCiAgICBtdiAvdG1wL3dnMC5jb25mLm92ZXJyaWRlIC90bXAvd2cwLmNvbmYKICBmaQogIHdnLXF1aWNrIHVwIC90bXAvd2cwLmNvbmYKZmkKZXhwb3J0IEhUVFBfUFJPWFk9aHR0cDovLzEwLjc3LjAuMTozMTI4CmV4cG9ydCBIVFRQU19QUk9YWT1odHRwOi8vMTAuNzcuMC4xOjMxMjgKZXhwb3J0IGh0dHBfcHJveHk9aHR0cDovLzEwLjc3LjAuMTozMTI4CmV4cG9ydCBodHRwc19wcm94eT1odHRwOi8vMTAuNzcuMC4xOjMxMjgKZXhwb3J0IE5PX1BST1hZPTEyNy4wLjAuMSxsb2NhbGhvc3QsLnN2YywuY2x1c3Rlci5sb2NhbCwxMC4wLjAuMC84LDEwMC42NC4wLjAvMTAsMTcyLjE2LjAuMC8xMiwxOTIuMTY4LjAuMC8xNiwubXlodWF3ZWljbG91ZC5jb20sLmh1YXdlaWNsb3VkLmNvbSwuYnVpbGRraXRlLmNvbSxidWlsZGtpdGVhcnRpZmFjdHMuY29tLC5idWlsZGtpdGVhcnRpZmFjdHMuY29tLC5hbWF6b25hd3MuY29tCmV4cG9ydCBub19wcm94eT0iJE5PX1BST1hZIgo=' | base64 -d > /hooks/environment; chmod 755 /hooks/environment" + volumeMounts: + - name: agent-hooks + mountPath: /hooks + containers: + - name: checkout + env: + - name: BUILDKITE_HOOKS_PATH + value: /buildkite/hooks + - name: AKERNEL_WG_ENDPOINT_OVERRIDE + value: "159.138.22.93:443" + securityContext: + capabilities: + add: ["NET_ADMIN"] + - name: container-0 + image: "swr.cn-southwest-2.myhuaweicloud.com/yuanrong-dev/sandbox-packager:v20260506_kubectl" + env: + - name: BUILDKITE_HOOKS_PATH + value: /buildkite/hooks + - name: AKERNEL_WG_ENDPOINT_OVERRIDE + value: "159.138.22.93:443" + securityContext: + capabilities: + add: ["NET_ADMIN"] + resources: + requests: { cpu: "1", memory: "1Gi" } + limits: { cpu: "2", memory: "2Gi" } + volumes: + - name: agent-hooks + emptyDir: {} + timeout_in_minutes: 10 diff --git a/.buildkite/scripts/build_and_push.sh b/.buildkite/scripts/build_and_push.sh new file mode 100755 index 0000000..0547f02 --- /dev/null +++ b/.buildkite/scripts/build_and_push.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="${AKERNEL_REPO_ROOT:-$(cd "${SCRIPT_DIR}/../.." && pwd)}" +# shellcheck source=docker_job_helpers.sh +source "${SCRIPT_DIR}/docker_job_helpers.sh" + +artifact_manifest="${YR_ARTIFACT_MANIFEST:-${ROOT}/artifacts/yuanrong/artifact-manifest.json}" +image_manifest="${AKERNEL_IMAGE_MANIFEST:-${ROOT}/artifacts/image/image-manifest.json}" +repository="${AKERNEL_IMAGE_REPOSITORY:-swr.cn-southwest-2.myhuaweicloud.com/openyuanrong/akernel-all-in-one}" +tag="${AKERNEL_IMAGE_TAG:-}" +include_kata="${AKERNEL_INCLUDE_KATA:-true}" +include_nvidia="${AKERNEL_INCLUDE_NVIDIA:-true}" +dockerd_log="${AKERNEL_DOCKERD_LOG:-${ROOT}/artifacts/image/dockerd.log}" +build_log="${AKERNEL_BUILD_LOG:-$(dirname "${image_manifest}")/build.log}" +secret_dir="" + +cleanup() { + docker_job_stop_dockerd + if [[ -n "${secret_dir}" ]]; then + rm -rf "${secret_dir}" + fi +} +trap cleanup EXIT + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +[[ -f "${artifact_manifest}" ]] || die "missing YuanRong artifact manifest: ${artifact_manifest}" +[[ "${repository}" =~ ^[A-Za-z0-9._:-]+(/[A-Za-z0-9._-]+)+$ ]] || \ + die "invalid image repository: ${repository}" +case "${include_kata}" in true|false) ;; *) die "AKERNEL_INCLUDE_KATA must be true or false" ;; esac +case "${include_nvidia}" in true|false) ;; *) die "AKERNEL_INCLUDE_NVIDIA must be true or false" ;; esac + +manifest_values="$(python3 - "${artifact_manifest}" <<'PY' +import json +import re +import sys + +path = sys.argv[1] +with open(path, encoding="utf-8") as stream: + data = json.load(stream) +if data.get("schema_version") != 1: + raise SystemExit("artifact manifest schema_version must be 1") +source = data.get("source") +if not isinstance(source, dict) or source.get("type") not in {"release", "buildkite"}: + raise SystemExit("artifact manifest source is invalid") +values = [] +for name, kinds in (("core", {"wheel"}), ("rrt", {"runtime", "wheel"})): + entry = data.get(name) + if not isinstance(entry, dict) or entry.get("kind") not in kinds: + raise SystemExit(f"artifact manifest {name} is invalid") + url = entry.get("url") + digest = entry.get("sha256") + if not isinstance(url, str) or not url.startswith(("http://", "https://")): + raise SystemExit(f"artifact manifest {name} URL is invalid") + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise SystemExit(f"artifact manifest {name} SHA-256 is invalid") + values.extend((entry["kind"], url, digest)) +source_reference = ( + source.get("version", "") + if source["type"] == "release" + else str(source.get("build_number", "")) +) +if not source_reference: + raise SystemExit("artifact manifest source reference is missing") +print("\t".join([source["type"], source_reference] + values)) +PY +)" || die "invalid YuanRong artifact manifest" + +IFS=$'\t' read -r \ + source_type source_reference \ + core_kind core_url core_sha256 \ + rrt_kind rrt_url rrt_sha256 <<<"${manifest_values}" +[[ "${core_kind}" == "wheel" ]] || die "YuanRong core artifact must be a wheel" + +branch="${BUILDKITE_BRANCH:-$(git -C "${ROOT}" branch --show-current)}" +commit="${BUILDKITE_COMMIT:-$(git -C "${ROOT}" rev-parse HEAD)}" +build_number="${BUILDKITE_BUILD_NUMBER:-local}" +[[ "${commit}" =~ ^[0-9A-Fa-f]{12,64}$ ]] || die "invalid AKernel commit: ${commit}" +branch_component="$( + printf '%s' "${branch}" | tr '[:upper:]' '[:lower:]' | \ + sed -E 's/[^a-z0-9_.-]+/-/g; s/^-+//; s/-+$//; s/-+/-/g' +)" +[[ -n "${branch_component}" ]] || branch_component=detached +if [[ "${source_type}" == "release" ]]; then + source_component="yr${source_reference}" +else + source_component="yrbk${source_reference}" +fi +if [[ -z "${tag}" ]]; then + identity_suffix="${build_number}-${commit:0:12}-${source_component}" + max_branch_length=$((128 - ${#identity_suffix} - 1)) + [[ "${max_branch_length}" -ge 1 ]] || die "generated image identity suffix is too long" + branch_component="${branch_component:0:${max_branch_length}}" + branch_component="${branch_component%-}" + [[ -n "${branch_component}" ]] || branch_component=detached + tag="${branch_component}-${identity_suffix}" +fi +[[ "${tag}" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]] || \ + die "invalid image tag: ${tag}" +image="${repository}:${tag}" +registry="${repository%%/*}" + +if [[ -n "${SWR_USERNAME:-}" || -n "${SWR_PASSWORD:-}" ]]; then + [[ -n "${SWR_USERNAME:-}" && -n "${SWR_PASSWORD:-}" ]] || \ + die "SWR credentials require both SWR_USERNAME and SWR_PASSWORD" +elif [[ -z "${SWR_DOCKER_CONFIG_JSON:-}" ]]; then + die "SWR credentials are required to push ${repository}" +fi + +mkdir -p \ + "$(dirname "${dockerd_log}")" \ + "$(dirname "${build_log}")" \ + "$(dirname "${image_manifest}")" +docker_job_start_dockerd "${dockerd_log}" + +git -C "${ROOT}" submodule sync -- src/sandboxd src/distill-fs +git -C "${ROOT}" submodule update --init --recursive --jobs=4 -- \ + src/sandboxd src/distill-fs + +build_arguments=( + build + "RUNTIME_PROFILE=rrt" + "IMAGE_REPOSITORY=${repository}" + "IMAGE_TAG=${tag}" + "OPEN_YR_CORE_WHEEL_URL=${core_url}" + "OPEN_YR_CORE_WHEEL_SHA256=${core_sha256}" + "AKERNEL_INCLUDE_KATA=${include_kata}" + "AKERNEL_INCLUDE_NVIDIA=${include_nvidia}" +) +if [[ "${source_type}" == "release" ]]; then + build_arguments+=("OPEN_YR_VERSION=${source_reference}") +fi +case "${rrt_kind}" in + runtime) + build_arguments+=( + "RRT_RUNTIME_URL=${rrt_url}" + "RRT_RUNTIME_SHA256=${rrt_sha256}" + ) + ;; + wheel) + build_arguments+=( + "OPEN_YR_RRT_WHEEL_URL=${rrt_url}" + "OPEN_YR_RRT_WHEEL_SHA256=${rrt_sha256}" + ) + ;; + *) die "unsupported RRT artifact kind: ${rrt_kind}" ;; +esac +if [[ -n "${PIP_INDEX_URL:-}" ]]; then + build_arguments+=("PIP_INDEX_URL=${PIP_INDEX_URL}") +fi +if [[ -n "${UV_PYTHON_INSTALL_MIRROR:-}" ]]; then + build_arguments+=("UV_PYTHON_INSTALL_MIRROR=${UV_PYTHON_INSTALL_MIRROR}") +fi + +set +e +(cd "${ROOT}" && make SHELL=/bin/bash "${build_arguments[@]}") 2>&1 | tee "${build_log}" +make_status="${PIPESTATUS[0]}" +set -e +[[ "${make_status}" -eq 0 ]] || die "AKernel image build failed with status ${make_status}" + +if [[ -n "${SWR_USERNAME:-}" ]]; then + printf '%s' "${SWR_PASSWORD}" | \ + "${DOCKER_BIN}" login "${registry}" -u "${SWR_USERNAME}" --password-stdin +else + secret_dir="$(mktemp -d)" + export DOCKER_CONFIG="${secret_dir}" + printf '%s' "${SWR_DOCKER_CONFIG_JSON}" >"${DOCKER_CONFIG}/config.json" + chmod 0600 "${DOCKER_CONFIG}/config.json" + python3 -m json.tool "${DOCKER_CONFIG}/config.json" >/dev/null || \ + die "SWR_DOCKER_CONFIG_JSON is not valid JSON" +fi + +"${DOCKER_BIN}" push "${image}" +inspect_json="$("${DOCKER_BIN}" manifest inspect --verbose "${image}")" +digest="$(python3 -c ' +import json, sys +data = json.load(sys.stdin) +found = [] +def walk(value): + if isinstance(value, dict): + digest = value.get("digest") + if isinstance(digest, str) and digest.startswith("sha256:"): + found.append(digest) + for child in value.values(): + walk(child) + elif isinstance(value, list): + for child in value: + walk(child) +walk(data) +print(found[0] if found else "") +' <<<"${inspect_json}")" +[[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]] || \ + die "registry did not return a valid digest for ${image}" + +temporary_manifest="${image_manifest}.tmp.$$" +python3 - \ + "${artifact_manifest}" "${temporary_manifest}" \ + "${repository}" "${tag}" "${digest}" \ + "${commit}" "${branch}" "${build_number}" \ + "${include_kata}" "${include_nvidia}" <<'PY' +import json +import sys + +( + artifact_path, + output_path, + repository, + tag, + digest, + commit, + branch, + build_number, + include_kata, + include_nvidia, +) = sys.argv[1:] +with open(artifact_path, encoding="utf-8") as stream: + artifacts = json.load(stream) +manifest = { + "schema_version": 1, + "image": { + "repository": repository, + "tag": tag, + "reference": f"{repository}:{tag}", + "digest": digest, + "digest_reference": f"{repository}@{digest}", + }, + "akernel": { + "commit": commit, + "branch": branch, + "build_number": build_number, + }, + "yuanrong": artifacts["source"], + "build": { + "runtime_profile": "rrt", + "include_kata": include_kata == "true", + "include_nvidia": include_nvidia == "true", + }, +} +with open(output_path, "w", encoding="utf-8") as stream: + json.dump(manifest, stream, indent=2, sort_keys=True) + stream.write("\n") +PY +mv "${temporary_manifest}" "${image_manifest}" +echo "pushed ${image}@${digest}" diff --git a/.buildkite/scripts/docker_job_helpers.sh b/.buildkite/scripts/docker_job_helpers.sh new file mode 100755 index 0000000..a7c20b6 --- /dev/null +++ b/.buildkite/scripts/docker_job_helpers.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# +# SPDX-License-Identifier: Apache-2.0 + +DOCKER_BIN="${DOCKER_BIN:-docker}" +DOCKERD_PID="${DOCKERD_PID:-}" + +docker_job_wait_for_docker() { + local timeout="${DOCKER_READY_TIMEOUT:-60}" + local elapsed=0 + while [[ "${elapsed}" -lt "${timeout}" ]]; do + if "${DOCKER_BIN}" info >/dev/null 2>&1; then + return 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + return 1 +} + +docker_job_stop_dockerd() { + if [[ -n "${DOCKERD_PID:-}" ]]; then + kill "${DOCKERD_PID}" >/dev/null 2>&1 || true + wait "${DOCKERD_PID}" >/dev/null 2>&1 || true + DOCKERD_PID="" + fi +} + +docker_job_start_dockerd() { + local log_file="${1:?dockerd log path required}" + local driver="${DOCKER_DRIVER:-overlay2}" + + if "${DOCKER_BIN}" info >/dev/null 2>&1; then + return 0 + fi + if ! command -v dockerd >/dev/null 2>&1; then + echo "Docker daemon is unavailable and dockerd is not installed." >&2 + return 1 + fi + + mkdir -p "$(dirname "${log_file}")" + : >"${log_file}" + dockerd \ + --host="${DOCKER_HOST:-unix:///var/run/docker.sock}" \ + --storage-driver="${driver}" >>"${log_file}" 2>&1 & + DOCKERD_PID="$!" + if docker_job_wait_for_docker; then + return 0 + fi + + docker_job_stop_dockerd + if [[ "${driver}" != "vfs" ]]; then + echo "dockerd with ${driver} did not become ready; retrying with vfs." >&2 + cp "${log_file}" "${log_file}.${driver}" 2>/dev/null || true + : >"${log_file}" + dockerd \ + --host="${DOCKER_HOST:-unix:///var/run/docker.sock}" \ + --storage-driver=vfs >>"${log_file}" 2>&1 & + DOCKERD_PID="$!" + if docker_job_wait_for_docker; then + return 0 + fi + docker_job_stop_dockerd + fi + + echo "Docker daemon failed to start. Log follows:" >&2 + cat "${log_file}" >&2 + return 1 +} diff --git a/.buildkite/scripts/package_deployments.py b/.buildkite/scripts/package_deployments.py new file mode 100755 index 0000000..d31c4af --- /dev/null +++ b/.buildkite/scripts/package_deployments.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Create AKernel standalone and Helm deployment bundles.""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import os +import pathlib +import re +import shutil +import sys +import tarfile +import tempfile +from typing import Any + + +class PackageError(RuntimeError): + pass + + +def load_json(path: pathlib.Path, description: str) -> dict[str, Any]: + if not path.is_file(): + raise PackageError(f"missing {description}: {path}") + with path.open(encoding="utf-8") as stream: + value = json.load(stream) + if not isinstance(value, dict) or value.get("schema_version") != 1: + raise PackageError(f"invalid {description}: schema_version must be 1") + return value + + +def file_sha256(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def normalized_tar_info(info: tarfile.TarInfo) -> tarfile.TarInfo: + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + info.mtime = 0 + return info + + +def create_archive(source: pathlib.Path, destination: pathlib.Path) -> None: + with destination.open("wb") as raw: + with gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as compressed: + with tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as archive: + archive.add( + source, + arcname=source.name, + recursive=True, + filter=normalized_tar_info, + ) + + +def copy_deployment_tree(source: pathlib.Path, destination: pathlib.Path) -> None: + if not source.is_dir(): + raise PackageError(f"missing deployment source directory: {source}") + shutil.copytree( + source, + destination, + ignore=shutil.ignore_patterns("data", "__pycache__", "*.pyc"), + ) + + +def validate_inputs( + artifact_manifest: dict[str, Any], + image_manifest: dict[str, Any], + sandbox_sdk: pathlib.Path, +) -> tuple[str, str, str, str]: + sdk = artifact_manifest.get("sandbox_sdk") + if not isinstance(sdk, dict) or sdk.get("kind") != "wheel": + raise PackageError("artifact manifest sandbox_sdk is invalid") + sdk_name = sdk.get("filename") + sdk_sha = sdk.get("sha256") + if not isinstance(sdk_name, str) or sandbox_sdk.name != sdk_name: + raise PackageError("sandbox SDK filename does not match artifact manifest") + if not isinstance(sdk_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", sdk_sha): + raise PackageError("sandbox SDK SHA-256 is invalid") + actual_sdk_sha = file_sha256(sandbox_sdk) + if actual_sdk_sha != sdk_sha: + raise PackageError( + f"sandbox SDK SHA-256 mismatch: expected {sdk_sha}, got {actual_sdk_sha}" + ) + + image = image_manifest.get("image") + if not isinstance(image, dict): + raise PackageError("image manifest image is invalid") + repository = image.get("repository") + tag = image.get("tag") + digest = image.get("digest") + reference = image.get("reference") + digest_reference = image.get("digest_reference") + if not isinstance(repository, str) or not re.fullmatch( + r"[A-Za-z0-9._:-]+(/[A-Za-z0-9._-]+)+", repository + ): + raise PackageError("image repository is invalid") + if not isinstance(tag, str) or not re.fullmatch( + r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}", tag + ): + raise PackageError("image tag is invalid") + if not isinstance(digest, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", digest): + raise PackageError("image digest is invalid") + if reference != f"{repository}:{tag}": + raise PackageError("image reference is inconsistent") + if digest_reference != f"{repository}@{digest}": + raise PackageError("image digest reference is inconsistent") + return repository, tag, digest, sdk_name + + +def write_common_files( + root: pathlib.Path, + artifact_manifest_path: pathlib.Path, + image_manifest_path: pathlib.Path, + sandbox_sdk: pathlib.Path, +) -> None: + manifests = root / "manifests" + artifacts = root / "artifacts" + manifests.mkdir(parents=True) + artifacts.mkdir(parents=True) + shutil.copyfile(artifact_manifest_path, manifests / "artifact-manifest.json") + shutil.copyfile(image_manifest_path, manifests / "image-manifest.json") + shutil.copyfile(sandbox_sdk, artifacts / sandbox_sdk.name) + + +def package( + repo_root: pathlib.Path, + artifact_manifest_path: pathlib.Path, + image_manifest_path: pathlib.Path, + sandbox_sdk: pathlib.Path, + output_dir: pathlib.Path, + targets: set[str], +) -> None: + if output_dir.exists(): + raise PackageError(f"output directory already exists: {output_dir}") + artifact_manifest = load_json(artifact_manifest_path, "artifact manifest") + image_manifest = load_json(image_manifest_path, "image manifest") + repository, tag, digest, _ = validate_inputs( + artifact_manifest, image_manifest, sandbox_sdk + ) + + output_dir.parent.mkdir(parents=True, exist_ok=True) + temporary = pathlib.Path( + tempfile.mkdtemp(prefix=f".{output_dir.name}.", dir=output_dir.parent) + ) + products = temporary / "products" + staging = temporary / "staging" + products.mkdir() + staging.mkdir() + try: + shutil.copyfile( + artifact_manifest_path, products / "artifact-manifest.json" + ) + shutil.copyfile(image_manifest_path, products / "image-manifest.json") + shutil.copyfile(sandbox_sdk, products / sandbox_sdk.name) + + if "standalone" in targets: + bundle_name = f"akernel-standalone-{tag}" + bundle_root = staging / bundle_name + copy_deployment_tree( + repo_root / "deploy" / "standalone", + bundle_root / "deploy" / "standalone", + ) + write_common_files( + bundle_root, + artifact_manifest_path, + image_manifest_path, + sandbox_sdk, + ) + (bundle_root / "image.env").write_text( + f"IMAGE={repository}:{tag}\n" + f"IMAGE_DIGEST={repository}@{digest}\n", + encoding="utf-8", + ) + create_archive( + bundle_root, products / f"akernel-standalone-{tag}.tar.gz" + ) + + if "helm" in targets: + bundle_name = f"akernel-helm-{tag}" + bundle_root = staging / bundle_name + copy_deployment_tree( + repo_root / "deploy" / "akernel", + bundle_root / "deploy" / "akernel", + ) + write_common_files( + bundle_root, + artifact_manifest_path, + image_manifest_path, + sandbox_sdk, + ) + (bundle_root / "values.image.yaml").write_text( + "core:\n" + " image:\n" + f" repository: {json.dumps(repository)}\n" + f" tag: {json.dumps(tag)}\n" + f"# immutable image: {repository}@{digest}\n", + encoding="utf-8", + ) + create_archive(bundle_root, products / f"akernel-helm-{tag}.tgz") + + checksum_lines = [] + for path in sorted(products.iterdir(), key=lambda item: item.name): + if path.is_file(): + checksum_lines.append(f"{file_sha256(path)} {path.name}") + (products / "SHA256SUMS").write_text( + "\n".join(checksum_lines) + "\n", encoding="utf-8" + ) + os.replace(products, output_dir) + finally: + shutil.rmtree(temporary, ignore_errors=True) + + +def parse_targets(value: str) -> set[str]: + values = [entry.strip() for entry in value.split(",") if entry.strip()] + targets = set(values) + if not targets: + raise argparse.ArgumentTypeError("at least one deployment target is required") + unknown = targets - {"standalone", "helm"} + if unknown: + raise argparse.ArgumentTypeError( + "unsupported deployment target(s): " + ", ".join(sorted(unknown)) + ) + return targets + + +def argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo-root", required=True, type=pathlib.Path) + parser.add_argument("--artifact-manifest", required=True, type=pathlib.Path) + parser.add_argument("--image-manifest", required=True, type=pathlib.Path) + parser.add_argument("--sandbox-sdk", required=True, type=pathlib.Path) + parser.add_argument("--output-dir", required=True, type=pathlib.Path) + parser.add_argument("--targets", required=True, type=parse_targets) + return parser + + +def main() -> int: + arguments = argument_parser().parse_args() + try: + package( + arguments.repo_root, + arguments.artifact_manifest, + arguments.image_manifest, + arguments.sandbox_sdk, + arguments.output_dir, + arguments.targets, + ) + except (PackageError, OSError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + print( + "packaged AKernel deployment target(s): " + + ", ".join(sorted(arguments.targets)) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.buildkite/scripts/resolve_yuanrong.py b/.buildkite/scripts/resolve_yuanrong.py new file mode 100755 index 0000000..30b713c --- /dev/null +++ b/.buildkite/scripts/resolve_yuanrong.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +"""Resolve YuanRong release or Buildkite artifacts into one manifest.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pathlib +import re +import shutil +import sys +import tempfile +import urllib.error +import urllib.parse +import urllib.request +from typing import Any + + +DEFAULT_RELEASE_BASE_URL = ( + "https://github.com/openYuanrong-mirror/yuanrong/releases/download" +) +DEFAULT_PYPI_BASE_URL = "https://pypi.org/pypi" +DEFAULT_BUILDKITE_API_URL = "https://api.buildkite.com/v2" +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") + + +class ResolutionError(RuntimeError): + pass + + +def request_bytes(url: str, token: str = "") -> bytes: + headers = {"User-Agent": "akernel-buildkite-artifact-resolver/1"} + if token: + headers["Authorization"] = f"Bearer {token}" + request = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(request, timeout=120) as response: + return response.read() + + +def request_json(url: str, token: str = "") -> dict[str, Any]: + try: + value = json.loads(request_bytes(url, token)) + except json.JSONDecodeError as error: + raise ResolutionError(f"invalid JSON response from {url}") from error + if not isinstance(value, dict): + raise ResolutionError(f"expected a JSON object from {url}") + return value + + +def digest_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def parse_published_sha256(value: bytes, filename: str) -> str: + try: + fields = value.decode("utf-8").strip().split() + except UnicodeDecodeError as error: + raise ResolutionError(f"invalid checksum file for {filename}") from error + if not fields or not SHA256_PATTERN.fullmatch(fields[0].lower()): + raise ResolutionError(f"invalid checksum file for {filename}") + if len(fields) > 1 and pathlib.PurePath(fields[-1].lstrip("*")) .name != filename: + raise ResolutionError(f"checksum file names the wrong artifact for {filename}") + return fields[0].lower() + + +def verified_download(url: str, expected_sha256: str = "") -> tuple[bytes, str]: + value = request_bytes(url) + actual = digest_bytes(value) + if expected_sha256 and actual != expected_sha256: + raise ResolutionError( + f"SHA-256 mismatch for {urllib.parse.unquote(url)}: " + f"expected {expected_sha256}, got {actual}" + ) + return value, actual + + +def artifact(filename: str, url: str, sha256: str, kind: str) -> dict[str, str]: + return { + "kind": kind, + "filename": filename, + "url": url, + "sha256": sha256, + } + + +def resolve_release( + version: str, release_base_url: str, pypi_base_url: str +) -> tuple[dict[str, Any], bytes]: + if not re.fullmatch(r"[0-9A-Za-z][0-9A-Za-z.+_-]*", version): + raise ResolutionError(f"invalid YuanRong release version: {version}") + + release_root = f"{release_base_url.rstrip('/')}/{urllib.parse.quote(version)}" + core_name = ( + f"openyuanrong_core-{version}-py3-none-manylinux_2_31_x86_64.whl" + ) + runtime_name = "rrt-runtime-amd64" + core_url = f"{release_root}/{core_name}" + runtime_url = f"{release_root}/{runtime_name}" + core_sha = parse_published_sha256( + request_bytes(f"{core_url}.sha256"), core_name + ) + runtime_sha = parse_published_sha256( + request_bytes(f"{runtime_url}.sha256"), runtime_name + ) + _, actual_core_sha = verified_download(core_url, core_sha) + _, actual_runtime_sha = verified_download(runtime_url, runtime_sha) + + pypi_url = ( + f"{pypi_base_url.rstrip('/')}/openyuanrong-sandbox/" + f"{urllib.parse.quote(version)}/json" + ) + pypi = request_json(pypi_url) + candidates = [] + for entry in pypi.get("urls", []): + if not isinstance(entry, dict) or entry.get("yanked"): + continue + filename = entry.get("filename") + if ( + entry.get("packagetype") == "bdist_wheel" + and isinstance(filename, str) + and filename.startswith("openyuanrong_sandbox-") + and filename.endswith("-py3-none-any.whl") + ): + candidates.append(entry) + if len(candidates) != 1: + raise ResolutionError( + "expected exactly one universal openyuanrong_sandbox wheel " + f"for release {version}, found {len(candidates)}" + ) + sdk_entry = candidates[0] + sdk_name = str(sdk_entry["filename"]) + sdk_url = str(sdk_entry.get("url", "")) + sdk_sha = str(sdk_entry.get("digests", {}).get("sha256", "")).lower() + if not sdk_url or not SHA256_PATTERN.fullmatch(sdk_sha): + raise ResolutionError(f"incomplete PyPI metadata for {sdk_name}") + sdk_bytes, actual_sdk_sha = verified_download(sdk_url, sdk_sha) + + return ( + { + "schema_version": 1, + "source": {"type": "release", "version": version}, + "core": artifact(core_name, core_url, actual_core_sha, "wheel"), + "rrt": artifact( + runtime_name, runtime_url, actual_runtime_sha, "runtime" + ), + "sandbox_sdk": artifact( + sdk_name, sdk_url, actual_sdk_sha, "wheel" + ), + }, + sdk_bytes, + ) + + +def parse_obs_urls(value: object, key: str) -> list[tuple[str, str]]: + if not isinstance(value, str): + raise ResolutionError(f"Buildkite metadata {key} must be a string") + entries: list[tuple[str, str]] = [] + for line in value.splitlines(): + if not line.strip(): + continue + fields = line.split("\t") + if len(fields) != 2 or not fields[0] or not fields[1]: + raise ResolutionError(f"malformed URL entry in Buildkite metadata {key}") + filename, url = fields + parsed = urllib.parse.urlparse(url) + if parsed.scheme not in {"http", "https"}: + raise ResolutionError(f"unsupported artifact URL in Buildkite metadata {key}") + url_name = pathlib.PurePosixPath(urllib.parse.unquote(parsed.path)).name + if url_name != filename: + raise ResolutionError(f"artifact filename/URL mismatch in metadata {key}") + entries.append((filename, url)) + return entries + + +def select_one( + entries: list[tuple[str, str]], pattern: str, description: str +) -> tuple[str, str]: + matches = [entry for entry in entries if re.fullmatch(pattern, entry[0])] + if len(matches) != 1: + raise ResolutionError( + f"expected exactly one {description} artifact, found {len(matches)}" + ) + return matches[0] + + +def resolve_buildkite( + organization: str, + pipeline: str, + build_number: int, + api_base_url: str, + api_token: str, +) -> tuple[dict[str, Any], bytes]: + if not api_token: + raise ResolutionError("a Buildkite API token with read_builds is required") + for label, value in (("organization", organization), ("pipeline", pipeline)): + if not re.fullmatch(r"[0-9A-Za-z][0-9A-Za-z_-]*", value): + raise ResolutionError(f"invalid Buildkite {label}: {value}") + if build_number <= 0: + raise ResolutionError("Buildkite build number must be positive") + + api_url = ( + f"{api_base_url.rstrip('/')}/organizations/{organization}/pipelines/" + f"{pipeline}/builds/{build_number}" + ) + build = request_json(api_url, api_token) + if build.get("state") != "passed": + raise ResolutionError( + f"YuanRong Buildkite build {pipeline}#{build_number} must be passed; " + f"state is {build.get('state', 'unknown')}" + ) + metadata = build.get("meta_data") + if not isinstance(metadata, dict): + raise ResolutionError("YuanRong Buildkite build has no metadata object") + + all_entries = parse_obs_urls( + metadata.get("obs-urls.build-all-amd64", ""), + "obs-urls.build-all-amd64", + ) + rrt_entries = parse_obs_urls( + metadata.get("obs-urls.build-rrt-amd64", ""), + "obs-urls.build-rrt-amd64", + ) + sdk_value = metadata.get("obs-urls.test-sandbox-sdk") + sdk_entries = ( + parse_obs_urls(sdk_value, "obs-urls.test-sandbox-sdk") + if sdk_value + else all_entries + ) + core_name, core_url = select_one( + all_entries, + r"openyuanrong_core-.*(?:x86_64|amd64)\.whl", + "openyuanrong_core", + ) + rrt_name, rrt_url = select_one( + rrt_entries, r"openyuanrong_rrt-.*\.whl", "openyuanrong_rrt" + ) + sdk_name, sdk_url = select_one( + sdk_entries, + r"openyuanrong_sandbox-.*-py3-none-any\.whl", + "openyuanrong_sandbox", + ) + + _, core_sha = verified_download(core_url) + _, rrt_sha = verified_download(rrt_url) + sdk_bytes, sdk_sha = verified_download(sdk_url) + source = { + "type": "buildkite", + "organization": organization, + "pipeline": pipeline, + "build_number": build_number, + "branch": str(build.get("branch", "")), + "commit": str(build.get("commit", "")), + } + return ( + { + "schema_version": 1, + "source": source, + "core": artifact(core_name, core_url, core_sha, "wheel"), + "rrt": artifact(rrt_name, rrt_url, rrt_sha, "wheel"), + "sandbox_sdk": artifact(sdk_name, sdk_url, sdk_sha, "wheel"), + }, + sdk_bytes, + ) + + +def write_output( + output_dir: pathlib.Path, manifest: dict[str, Any], sdk_bytes: bytes +) -> None: + if output_dir.exists(): + raise ResolutionError(f"output directory already exists: {output_dir}") + output_dir.parent.mkdir(parents=True, exist_ok=True) + temporary = pathlib.Path( + tempfile.mkdtemp(prefix=f".{output_dir.name}.", dir=output_dir.parent) + ) + try: + sdk_name = manifest["sandbox_sdk"]["filename"] + (temporary / sdk_name).write_bytes(sdk_bytes) + (temporary / "artifact-manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + os.replace(temporary, output_dir) + finally: + shutil.rmtree(temporary, ignore_errors=True) + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser(description=__doc__) + result.add_argument("--output-dir", required=True, type=pathlib.Path) + subparsers = result.add_subparsers(dest="source", required=True) + + release = subparsers.add_parser("release") + release.add_argument("--version", required=True) + release.add_argument("--release-base-url", default=DEFAULT_RELEASE_BASE_URL) + release.add_argument("--pypi-base-url", default=DEFAULT_PYPI_BASE_URL) + + buildkite = subparsers.add_parser("buildkite") + buildkite.add_argument("--organization", default="openyuanrong") + buildkite.add_argument("--pipeline", default="yuanrong-jcl") + buildkite.add_argument("--build-number", required=True, type=int) + buildkite.add_argument("--api-base-url", default=DEFAULT_BUILDKITE_API_URL) + buildkite.add_argument( + "--api-token", + default=( + os.environ.get("YR_BUILDKITE_API_TOKEN") + or os.environ.get("BUILDKITE_API_TOKEN") + or os.environ.get("BUILDKITE_PACKAGE_UPLOAD_TOKEN") + or "" + ), + ) + return result + + +def main() -> int: + arguments = parser().parse_args() + try: + if arguments.source == "release": + manifest, sdk_bytes = resolve_release( + arguments.version, + arguments.release_base_url, + arguments.pypi_base_url, + ) + else: + manifest, sdk_bytes = resolve_buildkite( + arguments.organization, + arguments.pipeline, + arguments.build_number, + arguments.api_base_url, + arguments.api_token, + ) + write_output(arguments.output_dir, manifest, sdk_bytes) + except (ResolutionError, urllib.error.URLError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + print( + f"resolved YuanRong {manifest['source']['type']} artifacts into " + f"{arguments.output_dir}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.buildkite/tests/test_build_and_push.py b/.buildkite/tests/test_build_and_push.py new file mode 100644 index 0000000..8a9d66c --- /dev/null +++ b/.buildkite/tests/test_build_and_push.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import json +import os +import pathlib +import stat +import subprocess +import tempfile +import textwrap +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SCRIPT = ROOT / ".buildkite" / "scripts" / "build_and_push.sh" +DIGEST = "sha256:" + "d" * 64 + + +def write_executable(path: pathlib.Path, body: str) -> None: + path.write_text(textwrap.dedent(body).lstrip(), encoding="utf-8") + path.chmod(path.stat().st_mode | stat.S_IXUSR) + + +class BuildAndPushTest(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.work = pathlib.Path(self.temporary.name) + self.fake_bin = self.work / "bin" + self.fake_bin.mkdir() + self.calls = self.work / "calls.log" + self.manifest = self.work / "artifact-manifest.json" + self.output = self.work / "image-manifest.json" + write_executable( + self.fake_bin / "git", + """ + #!/usr/bin/env bash + printf 'git %s\n' "$*" >>"${CALLS_LOG}" + case "$*" in + *'rev-parse HEAD'*) printf '%s\n' '0123456789abcdef0123456789abcdef01234567' ;; + *'branch --show-current'*) printf '%s\n' 'feature/fallback' ;; + esac + """, + ) + write_executable( + self.fake_bin / "make", + """ + #!/usr/bin/env bash + printf 'make %s\n' "$*" >>"${CALLS_LOG}" + """, + ) + write_executable( + self.fake_bin / "docker", + f""" + #!/usr/bin/env bash + printf 'docker %s\n' "$*" >>"${{CALLS_LOG}}" + case "$1" in + info) exit 0 ;; + login) IFS= read -r password; printf 'login-stdin-length=%s\n' "${{#password}}" >>"${{CALLS_LOG}}" ;; + manifest) + printf '%s\n' '{{"Descriptor":{{"digest":"{DIGEST}"}}}}' + ;; + esac + """, + ) + + def tearDown(self) -> None: + self.temporary.cleanup() + + def base_environment(self) -> dict[str, str]: + environment = os.environ.copy() + environment.update( + { + "PATH": f"{self.fake_bin}:{environment['PATH']}", + "CALLS_LOG": str(self.calls), + "YR_ARTIFACT_MANIFEST": str(self.manifest), + "AKERNEL_IMAGE_MANIFEST": str(self.output), + "AKERNEL_IMAGE_REPOSITORY": ( + "swr.cn-southwest-2.myhuaweicloud.com/" + "openyuanrong/akernel-all-in-one" + ), + "BUILDKITE_BRANCH": "feature/Pause_Resume", + "BUILDKITE_COMMIT": "abcdef0123456789abcdef0123456789abcdef01", + "BUILDKITE_BUILD_NUMBER": "17", + "SWR_USERNAME": "registry-user", + "SWR_PASSWORD": "secret-registry-password", + } + ) + return environment + + def run_script(self, environment: dict[str, str]): + return subprocess.run( + ["bash", str(SCRIPT)], + cwd=ROOT, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + def write_manifest(self, rrt_kind: str) -> None: + rrt_filename = ( + "rrt-runtime-amd64" + if rrt_kind == "runtime" + else "openyuanrong_rrt-0.7.0-py3-none-manylinux_x86_64.whl" + ) + self.manifest.write_text( + json.dumps( + { + "schema_version": 1, + "source": {"type": "release", "version": "0.9.7"}, + "core": { + "kind": "wheel", + "filename": "openyuanrong_core-0.9.7-x86_64.whl", + "url": "https://artifacts.example/core.whl", + "sha256": "a" * 64, + }, + "rrt": { + "kind": rrt_kind, + "filename": rrt_filename, + "url": "https://artifacts.example/rrt", + "sha256": "b" * 64, + }, + "sandbox_sdk": { + "kind": "wheel", + "filename": "openyuanrong_sandbox-0.9.7-py3-none-any.whl", + "url": "https://artifacts.example/sandbox.whl", + "sha256": "c" * 64, + }, + } + ), + encoding="utf-8", + ) + + def test_raw_runtime_build_uses_release_inputs_and_publishes_digest(self): + self.write_manifest("runtime") + result = self.run_script(self.base_environment()) + + self.assertEqual(result.returncode, 0, result.stderr) + calls = self.calls.read_text(encoding="utf-8").splitlines() + make_call = next(line for line in calls if line.startswith("make ")) + self.assertIn("SHELL=/bin/bash", make_call) + self.assertIn("RUNTIME_PROFILE=rrt", make_call) + self.assertIn("OPEN_YR_VERSION=0.9.7", make_call) + self.assertIn("OPEN_YR_CORE_WHEEL_URL=https://artifacts.example/core.whl", make_call) + self.assertIn("OPEN_YR_CORE_WHEEL_SHA256=" + "a" * 64, make_call) + self.assertIn("RRT_RUNTIME_URL=https://artifacts.example/rrt", make_call) + self.assertIn("RRT_RUNTIME_SHA256=" + "b" * 64, make_call) + self.assertNotIn("OPEN_YR_RRT_WHEEL_URL", make_call) + login_index = next(i for i, line in enumerate(calls) if line.startswith("docker login")) + push_index = next(i for i, line in enumerate(calls) if line.startswith("docker push")) + self.assertLess(login_index, push_index) + self.assertNotIn("secret-registry-password", "\n".join(calls)) + image = json.loads(self.output.read_text(encoding="utf-8")) + self.assertEqual( + image["image"]["reference"], + "swr.cn-southwest-2.myhuaweicloud.com/openyuanrong/" + "akernel-all-in-one:feature-pause_resume-17-abcdef012345-yr0.9.7", + ) + self.assertEqual(image["image"]["digest"], DIGEST) + self.assertEqual( + image["image"]["digest_reference"], + "swr.cn-southwest-2.myhuaweicloud.com/openyuanrong/" + f"akernel-all-in-one@{DIGEST}", + ) + self.assertEqual(image["yuanrong"]["type"], "release") + self.assertTrue((self.output.parent / "build.log").is_file()) + + def test_rrt_wheel_build_uses_wheel_override(self): + self.write_manifest("wheel") + manifest = json.loads(self.manifest.read_text(encoding="utf-8")) + manifest["source"] = { + "type": "buildkite", + "organization": "openyuanrong", + "pipeline": "yuanrong-jcl", + "build_number": 221, + "commit": "1" * 40, + } + self.manifest.write_text(json.dumps(manifest), encoding="utf-8") + result = self.run_script(self.base_environment()) + + self.assertEqual(result.returncode, 0, result.stderr) + make_call = next( + line + for line in self.calls.read_text(encoding="utf-8").splitlines() + if line.startswith("make ") + ) + self.assertIn("OPEN_YR_RRT_WHEEL_URL=https://artifacts.example/rrt", make_call) + self.assertIn("OPEN_YR_RRT_WHEEL_SHA256=" + "b" * 64, make_call) + self.assertNotIn("RRT_RUNTIME_URL", make_call) + image = json.loads(self.output.read_text(encoding="utf-8")) + self.assertTrue(image["image"]["tag"].endswith("-yrbk221")) + + def test_missing_registry_credentials_fails_before_build(self): + self.write_manifest("runtime") + environment = self.base_environment() + environment.pop("SWR_USERNAME") + environment.pop("SWR_PASSWORD") + result = self.run_script(environment) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("SWR credentials", result.stderr) + self.assertFalse(self.calls.exists()) + self.assertFalse(self.output.exists()) + + def test_invalid_explicit_tag_is_rejected(self): + self.write_manifest("runtime") + environment = self.base_environment() + environment["AKERNEL_IMAGE_TAG"] = "bad tag" + result = self.run_script(environment) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("invalid image tag", result.stderr) + self.assertFalse(self.output.exists()) + + def test_generated_tag_truncates_long_branch_but_preserves_identity_suffix(self): + self.write_manifest("runtime") + environment = self.base_environment() + environment["BUILDKITE_BRANCH"] = "feature/" + "x" * 200 + result = self.run_script(environment) + + self.assertEqual(result.returncode, 0, result.stderr) + image = json.loads(self.output.read_text(encoding="utf-8")) + tag = image["image"]["tag"] + self.assertLessEqual(len(tag), 128) + self.assertTrue(tag.endswith("-17-abcdef012345-yr0.9.7")) + + +if __name__ == "__main__": + unittest.main() diff --git a/.buildkite/tests/test_package_deployments.py b/.buildkite/tests/test_package_deployments.py new file mode 100644 index 0000000..b4daf06 --- /dev/null +++ b/.buildkite/tests/test_package_deployments.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import hashlib +import json +import pathlib +import subprocess +import sys +import tarfile +import tempfile +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +PACKAGER = ROOT / ".buildkite" / "scripts" / "package_deployments.py" +DIGEST = "sha256:" + "e" * 64 +TAG = "feature-pause-resume-17-abcdef012345-yrbk221" +REPOSITORY = ( + "swr.cn-southwest-2.myhuaweicloud.com/openyuanrong/akernel-all-in-one" +) + + +def sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +class PackageDeploymentsTest(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.work = pathlib.Path(self.temporary.name) + self.repo = self.work / "repo" + standalone = self.repo / "deploy" / "standalone" + chart = self.repo / "deploy" / "akernel" + (standalone / "config").mkdir(parents=True) + chart.mkdir(parents=True) + (standalone / "start.sh").write_text("#!/bin/bash\necho start\n") + (standalone / "config" / "config.json").write_text("{}\n") + (standalone / "data").mkdir() + (standalone / "data" / "secret-state").write_text("must not ship") + (chart / "Chart.yaml").write_text("apiVersion: v2\nname: akernel\n") + (chart / "values.yaml").write_text("core: {}\n") + + self.sdk_name = "openyuanrong_sandbox-0.10.1.dev47-py3-none-any.whl" + self.sdk = self.work / self.sdk_name + self.sdk_bytes = b"sandbox sdk wheel bytes" + self.sdk.write_bytes(self.sdk_bytes) + self.artifact_manifest = self.work / "artifact-manifest.json" + self.artifact_manifest.write_text( + json.dumps( + { + "schema_version": 1, + "source": { + "type": "buildkite", + "organization": "openyuanrong", + "pipeline": "yuanrong-jcl", + "build_number": 221, + "commit": "1" * 40, + }, + "core": { + "kind": "wheel", + "filename": "core.whl", + "url": "https://example.invalid/core.whl", + "sha256": "a" * 64, + }, + "rrt": { + "kind": "wheel", + "filename": "rrt.whl", + "url": "https://example.invalid/rrt.whl", + "sha256": "b" * 64, + }, + "sandbox_sdk": { + "kind": "wheel", + "filename": self.sdk_name, + "url": f"https://example.invalid/{self.sdk_name}", + "sha256": sha256(self.sdk_bytes), + }, + } + ) + ) + self.image_manifest = self.work / "image-manifest.json" + self.image_manifest.write_text( + json.dumps( + { + "schema_version": 1, + "image": { + "repository": REPOSITORY, + "tag": TAG, + "reference": f"{REPOSITORY}:{TAG}", + "digest": DIGEST, + "digest_reference": f"{REPOSITORY}@{DIGEST}", + }, + "akernel": { + "commit": "abcdef0123456789abcdef0123456789abcdef01", + "branch": "feature/Pause-Resume", + "build_number": "17", + }, + "yuanrong": { + "type": "buildkite", + "pipeline": "yuanrong-jcl", + "build_number": 221, + }, + "build": { + "runtime_profile": "rrt", + "include_kata": True, + "include_nvidia": True, + }, + } + ) + ) + self.output = self.work / "output" + + def tearDown(self) -> None: + self.temporary.cleanup() + + def run_packager(self, targets: str): + return subprocess.run( + [ + sys.executable, + str(PACKAGER), + "--repo-root", + str(self.repo), + "--artifact-manifest", + str(self.artifact_manifest), + "--image-manifest", + str(self.image_manifest), + "--sandbox-sdk", + str(self.sdk), + "--output-dir", + str(self.output), + "--targets", + targets, + ], + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + def archive_files(self, archive: pathlib.Path) -> dict[str, bytes]: + with tarfile.open(archive, "r:gz") as bundle: + result = {} + for member in bundle.getmembers(): + if member.isfile(): + stream = bundle.extractfile(member) + assert stream is not None + result[member.name] = stream.read() + return result + + def test_both_targets_include_sdk_manifests_and_pinned_image(self): + result = self.run_packager("standalone,helm") + + self.assertEqual(result.returncode, 0, result.stderr) + standalone_archive = self.output / f"akernel-standalone-{TAG}.tar.gz" + helm_archive = self.output / f"akernel-helm-{TAG}.tgz" + self.assertTrue(standalone_archive.is_file()) + self.assertTrue(helm_archive.is_file()) + + standalone_root = f"akernel-standalone-{TAG}" + standalone = self.archive_files(standalone_archive) + self.assertEqual( + standalone[f"{standalone_root}/deploy/standalone/start.sh"], + b"#!/bin/bash\necho start\n", + ) + self.assertNotIn( + f"{standalone_root}/deploy/standalone/data/secret-state", standalone + ) + self.assertEqual( + standalone[f"{standalone_root}/artifacts/{self.sdk_name}"], + self.sdk_bytes, + ) + self.assertEqual( + standalone[f"{standalone_root}/image.env"].decode(), + f"IMAGE={REPOSITORY}:{TAG}\nIMAGE_DIGEST={REPOSITORY}@{DIGEST}\n", + ) + self.assertIn( + f"{standalone_root}/manifests/artifact-manifest.json", standalone + ) + self.assertIn(f"{standalone_root}/manifests/image-manifest.json", standalone) + + helm_root = f"akernel-helm-{TAG}" + helm = self.archive_files(helm_archive) + self.assertEqual( + helm[f"{helm_root}/deploy/akernel/Chart.yaml"], + b"apiVersion: v2\nname: akernel\n", + ) + self.assertEqual( + helm[f"{helm_root}/values.image.yaml"].decode(), + "core:\n" + " image:\n" + f" repository: {json.dumps(REPOSITORY)}\n" + f" tag: {json.dumps(TAG)}\n" + f"# immutable image: {REPOSITORY}@{DIGEST}\n", + ) + self.assertEqual(helm[f"{helm_root}/artifacts/{self.sdk_name}"], self.sdk_bytes) + + for filename in ( + "artifact-manifest.json", + "image-manifest.json", + self.sdk_name, + ): + self.assertTrue((self.output / filename).is_file()) + checksum_lines = (self.output / "SHA256SUMS").read_text().splitlines() + expected_files = sorted( + path.name for path in self.output.iterdir() if path.name != "SHA256SUMS" + ) + self.assertEqual( + sorted(line.split(" ", 1)[1] for line in checksum_lines), expected_files + ) + for line in checksum_lines: + digest, filename = line.split(" ", 1) + self.assertEqual(digest, sha256((self.output / filename).read_bytes())) + + def test_target_filtering_emits_only_requested_bundle(self): + result = self.run_packager("helm") + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse((self.output / f"akernel-standalone-{TAG}.tar.gz").exists()) + self.assertTrue((self.output / f"akernel-helm-{TAG}.tgz").exists()) + + def test_sdk_digest_mismatch_leaves_no_output(self): + self.sdk.write_bytes(b"tampered wheel") + result = self.run_packager("standalone") + + self.assertNotEqual(result.returncode, 0) + self.assertIn("sandbox SDK SHA-256 mismatch", result.stderr) + self.assertFalse(self.output.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/.buildkite/tests/test_pipeline.py b/.buildkite/tests/test_pipeline.py new file mode 100644 index 0000000..9cd9ffd --- /dev/null +++ b/.buildkite/tests/test_pipeline.py @@ -0,0 +1,397 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import base64 +import os +import pathlib +import re +import subprocess +import tempfile +import unittest + +import yaml + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +GENERATOR = ROOT / ".buildkite" / "pipeline.sh" +BOOTSTRAP = ROOT / ".buildkite" / "pipeline.yml" +CACHE_PVC = ( + ROOT / ".buildkite" / "kubernetes" / "akernel-dependency-cache-pvc.yaml" +) + + +class PipelineTest(unittest.TestCase): + REQUIRED_NO_PROXY = { + "127.0.0.1", + "localhost", + ".svc", + ".cluster.local", + "10.0.0.0/8", + "100.64.0.0/10", + "172.16.0.0/12", + "192.168.0.0/16", + ".buildkite.com", + "buildkiteartifacts.com", + ".buildkiteartifacts.com", + ".amazonaws.com", + ".myhuaweicloud.com", + ".huaweicloud.com", + } + + def run_generator(self, **overrides: str): + environment = os.environ.copy() + for name in ( + "YR_SOURCE", + "YR_VERSION", + "YR_PIPELINE", + "YR_BUILD_NUMBER", + "YR_BUILDKITE_ORG", + "AKERNEL_DEPLOY_TARGETS", + "AKERNEL_IMAGE_REPOSITORY", + "AKERNEL_IMAGE_TAG", + "AKERNEL_INCLUDE_KATA", + "AKERNEL_INCLUDE_NVIDIA", + "AKERNEL_WG_ENDPOINT_OVERRIDE", + ): + environment.pop(name, None) + environment.update(overrides) + return subprocess.run( + ["bash", str(GENERATOR)], + cwd=ROOT, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + def parse_pipeline(self, result: subprocess.CompletedProcess[str]): + self.assertEqual(result.returncode, 0, result.stderr) + pipeline = yaml.safe_load(result.stdout) + self.assertIsInstance(pipeline, dict) + return pipeline + + def decode_egress_hook(self, step) -> str: + command = step["plugins"][0]["kubernetes"]["podSpecPatch"][ + "initContainers" + ][0]["command"][2] + match = re.search(r"printf '%s' '([A-Za-z0-9+/=]+)'", command) + self.assertIsNotNone(match, command) + return base64.b64decode(match.group(1)).decode("utf-8") + + def assert_restricted_egress( + self, step, expected_secrets=("AKERNEL_WG_CONFIG",) + ): + self.assertEqual(step["secrets"], list(expected_secrets)) + kubernetes = step["plugins"][0]["kubernetes"] + self.assertEqual( + kubernetes["extraVolumeMounts"], + [{"name": "agent-hooks", "mountPath": "/buildkite/hooks"}], + ) + pod = kubernetes["podSpecPatch"] + self.assertIn({"name": "agent-hooks", "emptyDir": {}}, pod["volumes"]) + + init = pod["initContainers"][0] + self.assertEqual(init["name"], "install-egress-hook") + hook = self.decode_egress_hook(step) + self.assertIn("if ! command -v wg", hook) + self.assertIn("mirrors.aliyun.com", hook) + self.assertIn("wireguard-tools iproute2 curl git make", hook) + self.assertIn("if ! ip link show wg0", hook) + self.assertIn('"$AKERNEL_WG_CONFIG" > /tmp/wg0.conf', hook) + self.assertIn("wg-quick up /tmp/wg0.conf", hook) + self.assertEqual(hook.count("wg-quick up /tmp/wg0.conf"), 1) + for variable in ("HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"): + self.assertIn( + f"export {variable}=http://10.77.0.1:3128", + hook, + ) + no_proxy = re.search(r"^export NO_PROXY=(.+)$", hook, re.MULTILINE) + self.assertIsNotNone(no_proxy, hook) + self.assertTrue( + self.REQUIRED_NO_PROXY.issubset(set(no_proxy.group(1).split(","))) + ) + self.assertIn('export no_proxy="$NO_PROXY"', hook) + + containers = {entry["name"]: entry for entry in pod["containers"]} + self.assertEqual(set(containers), {"checkout", "container-0"}) + for name in ("checkout", "container-0"): + container = containers[name] + self.assertIn( + {"name": "BUILDKITE_HOOKS_PATH", "value": "/buildkite/hooks"}, + container["env"], + ) + self.assertIn( + "NET_ADMIN", + container["securityContext"]["capabilities"]["add"], + ) + + self.assertIn("wg show wg0", step["command"]) + + def test_release_pipeline_builds_once_then_packages_both_targets(self): + pipeline = self.parse_pipeline( + self.run_generator(YR_SOURCE="release", YR_VERSION="0.9.7") + ) + + self.assertEqual(pipeline["checkout"]["submodules"], False) + steps = pipeline["steps"] + self.assertEqual( + [step["key"] for step in steps], + ["resolve-yuanrong", "build-image", "package-deployments"], + ) + self.assertNotIn("--api-token", steps[0]["command"]) + self.assertIn("artifact upload \"artifacts/yuanrong/*\"", steps[0]["command"]) + self.assertEqual(steps[0]["env"]["YR_SOURCE"], "release") + self.assertEqual(steps[0]["env"]["YR_VERSION"], "0.9.7") + self.assertEqual(steps[1]["depends_on"], "resolve-yuanrong") + self.assertEqual(steps[1]["env"]["UV_PYTHON_INSTALL_MIRROR"], "") + self.assertEqual(steps[2]["depends_on"], "build-image") + self.assertEqual( + steps[2]["env"]["AKERNEL_DEPLOY_TARGETS"], "standalone,helm" + ) + self.assertIn("artifact upload \"artifacts/packages/*\"", steps[2]["command"]) + + self.assert_restricted_egress( + steps[0], + ("AKERNEL_WG_CONFIG", "YR_BUILDKITE_API_TOKEN"), + ) + for step in steps[1:]: + self.assert_restricted_egress(step) + + pod = steps[1]["plugins"][0]["kubernetes"]["podSpecPatch"] + container = next( + entry for entry in pod["containers"] if entry["name"] == "container-0" + ) + self.assertTrue(container["securityContext"]["privileged"]) + self.assertEqual(container["resources"]["limits"]["cpu"], "10") + self.assertEqual(container["resources"]["limits"]["memory"], "32Gi") + docker_graph = next( + volume for volume in pod["volumes"] if volume["name"] == "docker-graph" + ) + self.assertEqual(docker_graph["emptyDir"]["sizeLimit"], "100Gi") + dependency_cache = next( + volume + for volume in pod["volumes"] + if volume["name"] == "dependency-cache" + ) + self.assertEqual( + dependency_cache["persistentVolumeClaim"]["claimName"], + "akernel-dependency-cache", + ) + self.assertIn( + { + "name": "dependency-cache", + "mountPath": "/var/cache/akernel-downloads", + }, + container["volumeMounts"], + ) + self.assertEqual( + steps[1]["env"]["AKERNEL_DEPENDENCY_CACHE_DIR"], + "/var/cache/akernel-downloads", + ) + + for isolated_step in (steps[0], steps[2]): + self.assertNotIn( + "AKERNEL_DEPENDENCY_CACHE_DIR", + isolated_step.get("env", {}), + ) + isolated_pod = isolated_step["plugins"][0]["kubernetes"]["podSpecPatch"] + self.assertNotIn( + "dependency-cache", + {volume["name"] for volume in isolated_pod["volumes"]}, + ) + for isolated_container in isolated_pod["containers"]: + self.assertNotIn( + "dependency-cache", + { + mount["name"] + for mount in isolated_container.get("volumeMounts", []) + }, + ) + secret_keys = { + ( + entry["name"], + entry["valueFrom"]["secretKeyRef"]["name"], + entry["valueFrom"]["secretKeyRef"]["key"], + ) + for entry in container["env"] + if "valueFrom" in entry + } + self.assertEqual( + secret_keys, + { + ("SWR_USERNAME", "swr-credentials", "username"), + ("SWR_PASSWORD", "swr-credentials", "password"), + ("SWR_DOCKER_CONFIG_JSON", "swr-pull-secret", ".dockerconfigjson"), + }, + ) + + def test_buildkite_pipeline_forwards_exact_build_without_embedding_token(self): + pipeline = self.parse_pipeline( + self.run_generator( + YR_SOURCE="buildkite", + YR_BUILDKITE_ORG="openyuanrong", + YR_PIPELINE="yuanrong-jcl", + YR_BUILD_NUMBER="221", + AKERNEL_DEPLOY_TARGETS="helm", + AKERNEL_IMAGE_TAG="manual-221", + ) + ) + + resolve = pipeline["steps"][0] + self.assertEqual( + resolve["env"], + { + "YR_SOURCE": "buildkite", + "YR_VERSION": "0.9.7", + "YR_BUILDKITE_ORG": "openyuanrong", + "YR_PIPELINE": "yuanrong-jcl", + "YR_BUILD_NUMBER": "221", + }, + ) + self.assertNotIn("token", resolve["command"].lower()) + build = pipeline["steps"][1] + self.assertEqual(build["env"]["AKERNEL_IMAGE_TAG"], "manual-221") + package = pipeline["steps"][2] + self.assertEqual(package["env"]["AKERNEL_DEPLOY_TARGETS"], "helm") + + def test_invalid_source_target_and_build_number_fail_before_yaml(self): + cases = ( + ({"YR_SOURCE": "filesystem"}, "YR_SOURCE"), + ( + {"YR_SOURCE": "buildkite", "YR_BUILD_NUMBER": "zero"}, + "YR_BUILD_NUMBER", + ), + ({"AKERNEL_DEPLOY_TARGETS": "standalone,vm"}, "deployment target"), + ) + for environment, expected in cases: + with self.subTest(environment=environment): + result = self.run_generator(**environment) + self.assertNotEqual(result.returncode, 0) + self.assertIn(expected, result.stderr) + self.assertEqual(result.stdout, "") + + def test_bootstrap_uploads_repository_owned_dynamic_pipeline(self): + pipeline = yaml.safe_load(BOOTSTRAP.read_text(encoding="utf-8")) + self.assertEqual(pipeline["checkout"]["submodules"], False) + self.assertEqual(len(pipeline["steps"]), 1) + step = pipeline["steps"][0] + self.assert_restricted_egress(step) + self.assertIn( + "bash .buildkite/pipeline.sh | buildkite-agent pipeline upload", + step["command"], + ) + self.assertIn("https://github.com/", step["command"]) + self.assertIn("https://example.com/", step["command"]) + self.assertIn('test "$$denied" = "403"', step["command"]) + self.assertEqual(step["agents"]["queue"], "default") + self.assertEqual(step["agents"]["arch"], "amd64") + + def test_wireguard_endpoint_override_reaches_checkout_and_commands(self): + generated = self.parse_pipeline(self.run_generator()) + bootstrap = yaml.safe_load(BOOTSTRAP.read_text(encoding="utf-8")) + steps = bootstrap["steps"] + generated["steps"] + + expected = { + "name": "AKERNEL_WG_ENDPOINT_OVERRIDE", + "value": "159.138.22.93:443", + } + for step in steps: + with self.subTest(step=step["label"]): + containers = step["plugins"][0]["kubernetes"]["podSpecPatch"][ + "containers" + ] + for container in containers: + self.assertIn(expected, container["env"]) + + def test_wireguard_endpoint_override_rewrites_config_before_start(self): + generated = self.parse_pipeline(self.run_generator()) + bootstrap = yaml.safe_load(BOOTSTRAP.read_text(encoding="utf-8")) + steps = bootstrap["steps"] + generated["steps"] + original_endpoint = "159.138.22.93:51820" + target_endpoint = "159.138.22.93:443" + + for step in steps: + with self.subTest(step=step["label"]), tempfile.TemporaryDirectory() as tmp: + temp = pathlib.Path(tmp) + binary_dir = temp / "bin" + binary_dir.mkdir() + for name, body in { + "wg": "#!/bin/sh\nexit 0\n", + "ip": "#!/bin/sh\nexit 1\n", + "wg-quick": "#!/bin/sh\ncat \"$2\"\n", + }.items(): + binary = binary_dir / name + binary.write_text(body, encoding="utf-8") + binary.chmod(0o755) + + config_path = temp / "wg0.conf" + hook = self.decode_egress_hook(step).replace( + "/tmp/wg0.conf", str(config_path) + ) + environment = os.environ.copy() + environment.update( + { + "PATH": f"{binary_dir}:/usr/bin:/bin", + "AKERNEL_WG_CONFIG": ( + "[Interface]\nPrivateKey = test-only\n" + "[Peer]\nPublicKey = test-only\n" + f"Endpoint = {original_endpoint}\n" + ), + "AKERNEL_WG_ENDPOINT_OVERRIDE": target_endpoint, + } + ) + result = subprocess.run( + ["/bin/sh"], + input=hook, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"Endpoint = {target_endpoint}", result.stdout) + self.assertNotIn(original_endpoint, result.stdout) + + def test_dependency_cache_uses_topology_aware_local_storage(self): + resources = { + resource["kind"]: resource + for resource in yaml.safe_load_all(CACHE_PVC.read_text(encoding="utf-8")) + } + self.assertEqual( + set(resources), {"StorageClass", "PersistentVolume", "PersistentVolumeClaim"} + ) + + storage_class = resources["StorageClass"] + self.assertEqual(storage_class["metadata"]["name"], "akernel-local-cache") + self.assertEqual(storage_class["provisioner"], "kubernetes.io/no-provisioner") + self.assertEqual(storage_class["volumeBindingMode"], "WaitForFirstConsumer") + self.assertEqual(storage_class["reclaimPolicy"], "Retain") + + volume = resources["PersistentVolume"] + self.assertEqual(volume["spec"]["storageClassName"], "akernel-local-cache") + self.assertEqual(volume["spec"]["persistentVolumeReclaimPolicy"], "Retain") + self.assertEqual( + volume["spec"]["local"]["path"], + "/mnt/paas/build-cache/akernel-dependency-cache", + ) + self.assertEqual( + volume["spec"]["nodeAffinity"]["required"]["nodeSelectorTerms"][0][ + "matchExpressions" + ][0]["values"], + ["10.10.189.4"], + ) + + claim = resources["PersistentVolumeClaim"] + self.assertNotIn("namespace", claim["metadata"]) + self.assertEqual(claim["spec"]["storageClassName"], "akernel-local-cache") + self.assertEqual(claim["spec"]["accessModes"], ["ReadWriteOnce"]) + self.assertEqual( + claim["spec"]["resources"]["requests"]["storage"], "10Gi" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.buildkite/tests/test_resolve_yuanrong.py b/.buildkite/tests/test_resolve_yuanrong.py new file mode 100644 index 0000000..1352bc4 --- /dev/null +++ b/.buildkite/tests/test_resolve_yuanrong.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import contextlib +import hashlib +import http.server +import json +import pathlib +import subprocess +import sys +import tempfile +import threading +import unittest +from urllib.parse import unquote, urlparse + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +RESOLVER = ROOT / ".buildkite" / "scripts" / "resolve_yuanrong.py" + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +class FixtureHandler(http.server.BaseHTTPRequestHandler): + fixtures: dict[str, tuple[int, str, bytes]] = {} + requests: list[tuple[str, str | None]] = [] + + def do_GET(self) -> None: + path = unquote(urlparse(self.path).path) + self.__class__.requests.append((path, self.headers.get("Authorization"))) + status, content_type, body = self.__class__.fixtures.get( + path, (404, "text/plain", b"not found") + ) + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, _format: str, *_args: object) -> None: + return + + +@contextlib.contextmanager +def fixture_server(fixtures: dict[str, tuple[int, str, bytes]]): + handler = type("PerTestFixtureHandler", (FixtureHandler,), {}) + handler.fixtures = fixtures + handler.requests = [] + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", handler + finally: + server.shutdown() + thread.join() + server.server_close() + + +class ResolveYuanRongTest(unittest.TestCase): + def run_resolver(self, output: pathlib.Path, *arguments: str): + return subprocess.run( + [sys.executable, str(RESOLVER), "--output-dir", str(output), *arguments], + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + def test_release_verifies_published_checksums_and_downloads_sandbox_sdk(self): + version = "0.9.7" + core_name = ( + "openyuanrong_core-0.9.7-py3-none-manylinux_2_31_x86_64.whl" + ) + runtime_name = "rrt-runtime-amd64" + sdk_name = "openyuanrong_sandbox-0.9.7-py3-none-any.whl" + core = b"release core wheel" + runtime = b"release rrt runtime" + sdk = b"release sandbox sdk" + fixtures: dict[str, tuple[int, str, bytes]] = { + f"/releases/download/{version}/{core_name}": ( + 200, + "application/octet-stream", + core, + ), + f"/releases/download/{version}/{core_name}.sha256": ( + 200, + "text/plain", + f"{sha256(core)} {core_name}\n".encode(), + ), + f"/releases/download/{version}/{runtime_name}": ( + 200, + "application/octet-stream", + runtime, + ), + f"/releases/download/{version}/{runtime_name}.sha256": ( + 200, + "text/plain", + f"{sha256(runtime)} {runtime_name}\n".encode(), + ), + f"/packages/{sdk_name}": (200, "application/octet-stream", sdk), + } + with tempfile.TemporaryDirectory() as temporary: + output = pathlib.Path(temporary) / "output" + with fixture_server(fixtures) as (base_url, handler): + pypi = { + "info": {"version": version}, + "urls": [ + { + "filename": sdk_name, + "packagetype": "bdist_wheel", + "url": f"{base_url}/packages/{sdk_name}", + "digests": {"sha256": sha256(sdk)}, + "yanked": False, + } + ], + } + handler.fixtures[f"/pypi/openyuanrong-sandbox/{version}/json"] = ( + 200, + "application/json", + json.dumps(pypi).encode(), + ) + result = self.run_resolver( + output, + "release", + "--version", + version, + "--release-base-url", + f"{base_url}/releases/download", + "--pypi-base-url", + f"{base_url}/pypi", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + manifest = json.loads((output / "artifact-manifest.json").read_text()) + self.assertEqual( + manifest, + { + "schema_version": 1, + "source": {"type": "release", "version": "0.9.7"}, + "core": { + "kind": "wheel", + "filename": core_name, + "url": f"{base_url}/releases/download/{version}/{core_name}", + "sha256": sha256(core), + }, + "rrt": { + "kind": "runtime", + "filename": runtime_name, + "url": f"{base_url}/releases/download/{version}/{runtime_name}", + "sha256": sha256(runtime), + }, + "sandbox_sdk": { + "kind": "wheel", + "filename": sdk_name, + "url": f"{base_url}/packages/{sdk_name}", + "sha256": sha256(sdk), + }, + }, + ) + self.assertEqual((output / sdk_name).read_bytes(), sdk) + + def test_buildkite_selects_exact_passed_build_metadata(self): + core_name = ( + "openyuanrong_core-0.7.0+abc-py3-none-manylinux_2_31_x86_64.whl" + ) + rrt_name = ( + "openyuanrong_rrt-0.7.0+abc-py3-none-manylinux_2_31_x86_64.whl" + ) + sdk_name = "openyuanrong_sandbox-0.10.1.dev47-py3-none-any.whl" + core = b"buildkite core" + rrt = b"buildkite rrt" + sdk = b"buildkite sandbox" + with tempfile.TemporaryDirectory() as temporary: + output = pathlib.Path(temporary) / "output" + with fixture_server({}) as (base_url, handler): + handler.fixtures.update( + { + f"/obs/{core_name}": (200, "application/octet-stream", core), + f"/obs/{rrt_name}": (200, "application/octet-stream", rrt), + f"/obs/{sdk_name}": (200, "application/octet-stream", sdk), + } + ) + build = { + "number": 221, + "state": "passed", + "branch": "codex/pause-resume-v1-package", + "commit": "ae33db1cf00ecf83fd58b8465daad9a1a0d4ac96", + "meta_data": { + "obs-urls.build-all-amd64": ( + f"{core_name}\t{base_url}/obs/{core_name}\n" + ), + "obs-urls.build-rrt-amd64": ( + f"{rrt_name}\t{base_url}/obs/{rrt_name}\n" + ), + "obs-urls.test-sandbox-sdk": ( + f"{sdk_name}\t{base_url}/obs/{sdk_name}\n" + ), + }, + } + api_path = "/v2/organizations/openyuanrong/pipelines/yuanrong-jcl/builds/221" + handler.fixtures[api_path] = ( + 200, + "application/json", + json.dumps(build).encode(), + ) + result = self.run_resolver( + output, + "buildkite", + "--organization", + "openyuanrong", + "--pipeline", + "yuanrong-jcl", + "--build-number", + "221", + "--api-base-url", + f"{base_url}/v2", + "--api-token", + "secret-test-token", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn((api_path, "Bearer secret-test-token"), handler.requests) + manifest = json.loads((output / "artifact-manifest.json").read_text()) + self.assertEqual( + manifest["source"], + { + "type": "buildkite", + "organization": "openyuanrong", + "pipeline": "yuanrong-jcl", + "build_number": 221, + "branch": "codex/pause-resume-v1-package", + "commit": "ae33db1cf00ecf83fd58b8465daad9a1a0d4ac96", + }, + ) + self.assertEqual(manifest["core"]["sha256"], sha256(core)) + self.assertEqual(manifest["rrt"]["kind"], "wheel") + self.assertEqual(manifest["rrt"]["sha256"], sha256(rrt)) + self.assertEqual(manifest["sandbox_sdk"]["filename"], sdk_name) + self.assertEqual((output / sdk_name).read_bytes(), sdk) + + def test_buildkite_rejects_non_passed_build_without_leaking_token(self): + token = "secret-token-must-not-appear" + build = { + "number": 222, + "state": "failed", + "branch": "main", + "commit": "deadbeef", + "meta_data": {}, + } + with tempfile.TemporaryDirectory() as temporary: + output = pathlib.Path(temporary) / "output" + with fixture_server({}) as (base_url, handler): + handler.fixtures[ + "/v2/organizations/openyuanrong/pipelines/yuanrong-jcl/builds/222" + ] = (200, "application/json", json.dumps(build).encode()) + result = self.run_resolver( + output, + "buildkite", + "--organization", + "openyuanrong", + "--pipeline", + "yuanrong-jcl", + "--build-number", + "222", + "--api-base-url", + f"{base_url}/v2", + "--api-token", + token, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("must be passed", result.stderr) + self.assertNotIn(token, result.stderr) + self.assertFalse((output / "artifact-manifest.json").exists()) + + def test_buildkite_rejects_duplicate_matching_artifacts(self): + core_name = ( + "openyuanrong_core-0.7.0-py3-none-manylinux_2_31_x86_64.whl" + ) + build = { + "number": 223, + "state": "passed", + "branch": "main", + "commit": "cafebabe", + "meta_data": { + "obs-urls.build-all-amd64": ( + f"{core_name}\thttps://example.invalid/one/{core_name}\n" + f"{core_name}\thttps://example.invalid/two/{core_name}\n" + ), + "obs-urls.build-rrt-amd64": "", + "obs-urls.test-sandbox-sdk": "", + }, + } + with tempfile.TemporaryDirectory() as temporary: + output = pathlib.Path(temporary) / "output" + with fixture_server({}) as (base_url, handler): + handler.fixtures[ + "/v2/organizations/openyuanrong/pipelines/yuanrong-jcl/builds/223" + ] = (200, "application/json", json.dumps(build).encode()) + result = self.run_resolver( + output, + "buildkite", + "--organization", + "openyuanrong", + "--pipeline", + "yuanrong-jcl", + "--build-number", + "223", + "--api-base-url", + f"{base_url}/v2", + "--api-token", + "test-token", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("exactly one openyuanrong_core", result.stderr) + self.assertFalse((output / "artifact-manifest.json").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/.gitmodules b/.gitmodules index 97ae8b8..dd4cc89 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "src/sandboxd"] path = src/sandboxd - url = https://github.com/inclusionAI/sandboxd.git + url = https://github.com/Chamberlain1998/sandboxd.git [submodule "src/distill-fs"] path = src/distill-fs url = https://github.com/inclusionAI/distill-fs.git diff --git a/AGENTS.md b/AGENTS.md index 0152490..f818bcf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,6 +17,11 @@ leading-wildcard DNS names. Experimental whole-device NVIDIA GPU and configurable writable-storage requests currently require runsc. +Kubernetes pause/resume is disabled by default. It requires an RRT-profile +all-in-one image and `core.pauseResume.enabled=true` in the Helm release. Use a +deployment-specific `node.hostDiskPath` when validating alongside another +AKernel release so checkpoint and sandboxd state never share a host directory. + Use AKernel when a task needs an isolated remote environment with command execution, file operations, interactive PTYs, port forwarding, or reverse tunnels. The project overview and deployment quick start are in @@ -33,6 +38,8 @@ tunnels. The project overview and deployment quick start are in - `sdk/python/tests/` - maintained AKernel SDK tests. - `src/yuanrong/` - pinned openYuanRong mirror checkout, including its recursive component submodules. +- `.buildkite/` - universal image pipeline, YuanRong artifact resolver, + deployment packager, and their behavior tests. - `builder/` - Dockerfiles, service configs, runtime rootfs build, and image entrypoint scripts for the public all-in-one image. - `deploy/` - Helm charts, standalone scripts, Terraform modules, and @@ -68,6 +75,7 @@ make token TTL=24h make print-env make sdk-test make deploy-script-check +make buildkite-check make e2e ``` @@ -113,6 +121,13 @@ For a build that will be pushed and deployed, set `IMAGE_REPOSITORY` and override on `make build` does not update the profile consumed by `make push`. The build creates only the selected image reference; it does not add a second `akernel-all-in-one` alias. `make push` pushes that selected reference directly. +Kata remains included by default. Set `AKERNEL_INCLUDE_KATA=false` only for a +runsc-only image, such as standalone validation on a host without KVM; this +skips the large Kata release payload and leaves the optional Kata runtime +binary absent, so sandboxd does not advertise it. +NVIDIA userspace tools also remain included by default. Set +`AKERNEL_INCLUDE_NVIDIA=false` for CPU-only builds when the NVIDIA package +repository is unavailable; images built this way cannot serve GPU sandboxes. The build helper performs two Docker builds. `builder/runtime.Dockerfile` creates `yr-runtime-rootfs.img`; the default `rrt` profile contains the @@ -143,6 +158,37 @@ provide both `OPEN_YR_CORE_WHEEL_URL` and `OPEN_YR_CORE_WHEEL_SHA256` to `make build`. The complete wheel is verified before it replaces the pinned release control plane. +This branch sets the core and RRT wheel URL/SHA pairs in the `Makefile` to +durable GitHub Release mirrors of the amd64 OBS artifacts from YuanRong +Buildkite build `yuanrong-jcl#237` at commit +`87cba622b491f7303415d179f3fa2fdae98e69e7`. Consequently, an unqualified +`make build` consumes that integrated build without a Buildkite token. Callers +can still override either URL/SHA pair together through the existing make +variables. + +The Dockerfiles delegate openYuanRong artifact acquisition to two stable, +replaceable build-context scripts: + +- `builder/downloaders/download-openyuanrong-core.sh` writes exactly one core + wheel to the destination directory passed as its only argument. +- `builder/downloaders/download-openyuanrong-rrt.sh` writes the unpacked RRT + executable to the destination file passed as its only argument. + +The checked-in implementations retain the checksum-pinned release downloads +and the existing URL/SHA override pairs. A private pipeline may replace either +script before `docker build` to fetch from OBS or another authenticated source +without patching a Dockerfile. Replacement scripts are build code: they must +authenticate their source where required, verify artifact integrity, and honor +the same output contract. Docker still validates that the core result is one +wheel and that the RRT result is an executable x86-64 ELF. + +The pause/resume process-script helper applies patches only to its exact known +legacy core-wheel SHA values. In default `auto` mode it accepts packages that +already provide the required contract and leaves other unknown packages +unchanged, so selecting a newer Buildkite wheel does not accidentally apply an +old source patch. Use its `require` mode when validating a package that must +provide that legacy standalone process contract. + Inspect the selected local versions without building an image: ```bash @@ -153,6 +199,15 @@ The final image uses standard OCI labels for the AKernel version and revision. Component semantic versions are reported by their binaries, and their exact source revisions are traceable through the AKernel commit's submodule gitlinks. +The repository-owned Buildkite pipeline is documented in +`.buildkite/README.md`. It builds one universal image and treats standalone +and Helm as deployment bundle selections, not different images. YuanRong +inputs may come from a checksum-published release or from the `obs-urls.*` +metadata of a selected passed YuanRong Buildkite build. Cross-pipeline API and +SWR credentials must come from Buildkite agent or Kubernetes secret +environments; never add them to pipeline YAML or build inputs. Run +`make buildkite-check` after changing pipeline scripts or artifact contracts. + ## Deploy Use [`deploy/README.md`](./deploy/README.md) as the deployment entry point. @@ -303,6 +358,12 @@ No separate `AKERNEL_GATEWAY_ADDRESS` is required for the default standalone layout. Standalone uses `akerneldev/all-in-one:latest` by default; pass `IMAGE` to test a locally built or differently tagged image. +Kubernetes deployments that expose Frontend control traffic and SandboxRouter +on separate listeners should set `AKERNEL_SERVER_ADDRESS` to Frontend and +`AKERNEL_SANDBOX_ROUTER_ADDRESS` to the direct SandboxRouter listener. The +port-specific variable affects only `Sandbox.get_port_url()`; PTY, file, and +reverse-tunnel traffic retain the Frontend/gateway endpoint. + Standalone GPU testing additionally requires NVIDIA Container Toolkit on the host and `AKERNEL_ENABLE_GPU=true`. sandboxd uses the read-only cgroup node-resource provider in standalone mode; Kubernetes deployments retain the diff --git a/Makefile b/Makefile index cc5cda6..e94d106 100644 --- a/Makefile +++ b/Makefile @@ -9,9 +9,22 @@ ENV ?= default IMAGE_TAG ?= IMAGE_REPOSITORY ?= GVISOR_RELEASE ?= +GVISOR_AMD64_SHA512 ?= GVISOR_RELEASE_BASE_URL ?= -OPEN_YR_CORE_WHEEL_URL ?= -OPEN_YR_CORE_WHEEL_SHA256 ?= +OTELCOL_CONTRIB_VERSION ?= +OTELCOL_CONTRIB_SHA256 ?= +OTELCOL_CONTRIB_URL ?= +OPEN_YR_VERSION ?= +OPEN_YR_CORE_WHEEL_URL ?= https://github.com/Chamberlain1998/AKernel/releases/download/yuanrong-buildkite-237-87cba622b491/openyuanrong_core-0.7.0%2B87cba622b491-py3-none-manylinux_2_31_x86_64.whl +OPEN_YR_CORE_WHEEL_SHA256 ?= 9eb44e1ea59153ab9a65a81fc32450c09376e835732290046d028cec2db3b200 +OPEN_YR_RRT_WHEEL_URL ?= https://github.com/Chamberlain1998/AKernel/releases/download/yuanrong-buildkite-237-87cba622b491/openyuanrong_rrt-0.7.0%2B87cba622b491-py3-none-manylinux_2_31_x86_64.whl +OPEN_YR_RRT_WHEEL_SHA256 ?= 3aff1b4a676ca28992a2478adab900bc7bd1e76928cc12016ae50fea412a68c4 +RRT_RUNTIME_URL ?= +RRT_RUNTIME_SHA256 ?= +PIP_INDEX_URL ?= +UV_PYTHON_INSTALL_MIRROR ?= +AKERNEL_INCLUDE_KATA ?= true +AKERNEL_INCLUDE_NVIDIA ?= true TOKEN_TTL ?= $(if $(TTL),$(TTL),24h) TENANT ?= default ROLE ?= developer @@ -49,7 +62,13 @@ help: @echo " make config INSTALL_DRAGONFLY=true Enable optional P2P image distribution" @echo " make build IMAGE_TAG= Build the all-in-one image" @echo " make build RUNTIME_PROFILE=python Include optional Python runtimes" + @echo " make build OPEN_YR_VERSION= Select a YuanRong release version" @echo " make build GVISOR_RELEASE= Override the pinned official gVisor tag" + @echo " GVISOR_AMD64_SHA512= gVisor version and digest must be overridden together" + @echo " make build OTELCOL_CONTRIB_VERSION= OTELCOL_CONTRIB_SHA256=" + @echo " OTel version and digest must be overridden together" + @echo " make build AKERNEL_INCLUDE_KATA=false Build a runsc-only image" + @echo " make build AKERNEL_INCLUDE_NVIDIA=false Build without NVIDIA userspace tools" @echo " make versions Show locally selected component versions" @echo " make push Push the configured all-in-one image" @echo " make plan Terraform plan" @@ -58,6 +77,7 @@ help: @echo " make print-env Print SDK environment exports" @echo " make sdk-check Lint, type-check, and test the Python SDK" @echo " make deploy-script-check Check deployment script syntax" + @echo " make buildkite-check Check the AKernel image pipeline" @echo " make e2e Run the basic SDK e2e example" @echo " make destroy Destroy cloud resources" @@ -98,9 +118,22 @@ build: if [[ -n "$(IMAGE_TAG)" ]]; then args+=(--tag "$(IMAGE_TAG)"); fi; \ if [[ -n "$(RUNTIME_PROFILE)" ]]; then args+=(--runtime-profile "$(RUNTIME_PROFILE)"); fi; \ if [[ -n "$(GVISOR_RELEASE)" ]]; then args+=(--gvisor-release "$(GVISOR_RELEASE)"); fi; \ + if [[ -n "$(GVISOR_AMD64_SHA512)" ]]; then args+=(--gvisor-amd64-sha512 "$(GVISOR_AMD64_SHA512)"); fi; \ if [[ -n "$(GVISOR_RELEASE_BASE_URL)" ]]; then args+=(--gvisor-release-base-url "$(GVISOR_RELEASE_BASE_URL)"); fi; \ + if [[ -n "$(OTELCOL_CONTRIB_VERSION)" ]]; then args+=(--otelcol-contrib-version "$(OTELCOL_CONTRIB_VERSION)"); fi; \ + if [[ -n "$(OTELCOL_CONTRIB_SHA256)" ]]; then args+=(--otelcol-contrib-sha256 "$(OTELCOL_CONTRIB_SHA256)"); fi; \ + if [[ -n "$(OTELCOL_CONTRIB_URL)" ]]; then args+=(--otelcol-contrib-url "$(OTELCOL_CONTRIB_URL)"); fi; \ + if [[ -n "$(OPEN_YR_VERSION)" ]]; then args+=(--open-yr-version "$(OPEN_YR_VERSION)"); fi; \ if [[ -n "$(OPEN_YR_CORE_WHEEL_URL)" ]]; then args+=(--open-yr-core-wheel-url "$(OPEN_YR_CORE_WHEEL_URL)"); fi; \ if [[ -n "$(OPEN_YR_CORE_WHEEL_SHA256)" ]]; then args+=(--open-yr-core-wheel-sha256 "$(OPEN_YR_CORE_WHEEL_SHA256)"); fi; \ + if [[ -n "$(OPEN_YR_RRT_WHEEL_URL)" ]]; then args+=(--open-yr-rrt-wheel-url "$(OPEN_YR_RRT_WHEEL_URL)"); fi; \ + if [[ -n "$(OPEN_YR_RRT_WHEEL_SHA256)" ]]; then args+=(--open-yr-rrt-wheel-sha256 "$(OPEN_YR_RRT_WHEEL_SHA256)"); fi; \ + if [[ -n "$(RRT_RUNTIME_URL)" ]]; then args+=(--rrt-runtime-url "$(RRT_RUNTIME_URL)"); fi; \ + if [[ -n "$(RRT_RUNTIME_SHA256)" ]]; then args+=(--rrt-runtime-sha256 "$(RRT_RUNTIME_SHA256)"); fi; \ + if [[ -n "$(PIP_INDEX_URL)" ]]; then args+=(--pip-index-url "$(PIP_INDEX_URL)"); fi; \ + if [[ -n "$(UV_PYTHON_INSTALL_MIRROR)" ]]; then args+=(--uv-python-install-mirror "$(UV_PYTHON_INSTALL_MIRROR)"); fi; \ + args+=(--include-kata "$(AKERNEL_INCLUDE_KATA)"); \ + args+=(--include-nvidia "$(AKERNEL_INCLUDE_NVIDIA)"); \ ./deploy/scripts/build-image.sh "$${args[@]}" .PHONY: versions @@ -169,6 +202,15 @@ deploy-script-check: xargs -0 -r python3 -c \ 'import pathlib, sys; [compile(pathlib.Path(path).read_bytes(), path, "exec") for path in sys.argv[1:]]' +.PHONY: buildkite-check +buildkite-check: + @python3 -m unittest discover -s .buildkite/tests -p 'test_*.py' -v + @find .buildkite -type f -name '*.py' -print0 | \ + xargs -0 python3 -m py_compile + @while IFS= read -r -d '' script; do \ + bash -n "$$script"; \ + done < <(find .buildkite -type f -name '*.sh' -print0) + .PHONY: destroy destroy: @args=(--vendor "$(VENDOR)" --env "$(ENV)"); \ diff --git a/README.md b/README.md index bea42e6..9d671e9 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ Built-in OpenTelemetry (OTEL) integration provides complete observability out of - 💡 [Examples](./sdk/python/examples/) - AKernel SDK examples and use cases - 🏗️ [Architecture](#architecture) - System design and components - 🚀 [Deployment](./deploy/README.md) - Installation and configuration guide +- 📦 [Buildkite Images](./.buildkite/README.md) - Reproducible image and deployment artifacts ### Bootstrap a Cluster @@ -102,6 +103,19 @@ make deploy See the [Deployment Guide](./deploy/README.md) for prerequisites, cloud-specific configuration, deployment verification, and cluster cleanup, and the [Build Guide](./CLAUDE.md) for development details. +The repository-owned [Buildkite image pipeline](./.buildkite/README.md) can +consume either a checksum-published YuanRong release or the artifacts from a +selected passed YuanRong Buildkite build. It pushes one universal image and +publishes the YuanRong sandbox SDK plus standalone and Helm deployment +bundles. + +This branch defaults source builds to durable, checksum-pinned mirrors of the +core and RRT wheels from YuanRong Buildkite build +[`yuanrong-jcl#237`](https://buildkite.com/openyuanrong/yuanrong-jcl/builds/237). +The normal `make build` command consumes the public GitHub Release assets +without extra credentials; set the matching `OPEN_YR_*_WHEEL_URL` and +`OPEN_YR_*_WHEEL_SHA256` pairs to override them. + ### Create a Sandbox diff --git a/builder/downloaders/cache-verified-download.sh b/builder/downloaders/cache-verified-download.sh new file mode 100755 index 0000000..737c5fe --- /dev/null +++ b/builder/downloaders/cache-verified-download.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ "$#" -ne 3 ]]; then + echo "usage: $0 URL DIGEST DESTINATION" >&2 + exit 2 +fi + +url="$1" +expected_digest="$2" +destination="$3" + +die() { + echo "$1" >&2 + exit 2 +} + +case "${url}" in + http://*|https://*) ;; + *) + die "cache URL must use http or https: ${url}" + ;; +esac +case "${expected_digest}" in + (*[!0-9a-f]*) die "cache digest must use lowercase hexadecimal characters" ;; +esac +case "${#expected_digest}" in + 64) checksum_command=sha256sum ;; + 128) checksum_command=sha512sum ;; + *) die "cache digest must be a SHA-256 or SHA-512 hexadecimal value" ;; +esac +if [[ -z "${destination}" || -d "${destination}" ]]; then + die "cache destination must be a file path: ${destination}" +fi + +if [[ -f "${destination}" ]] && + printf '%s %s\n' "${expected_digest}" "${destination}" | + "${checksum_command}" -c - >/dev/null 2>&1; then + printf 'cache-hit %s\n' "${destination}" + exit 0 +fi + +mkdir -p "$(dirname "${destination}")" +temporary="${destination}.part.${BUILDKITE_BUILD_ID:-local}.$$" +cleanup() { + rm -f -- "${temporary}" +} +trap cleanup EXIT + +curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ + "${url}" -o "${temporary}" +printf '%s %s\n' "${expected_digest}" "${temporary}" | + "${checksum_command}" -c - >/dev/null +chmod 0444 "${temporary}" +mv -f "${temporary}" "${destination}" +trap - EXIT + +printf 'cache-fill %s\n' "${destination}" diff --git a/builder/downloaders/download-openyuanrong-core.sh b/builder/downloaders/download-openyuanrong-core.sh new file mode 100755 index 0000000..43ffcfc --- /dev/null +++ b/builder/downloaders/download-openyuanrong-core.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ "$#" -ne 1 ]]; then + echo "usage: $0 DEST_DIR" >&2 + exit 2 +fi + +destination_dir="$1" +override_url="${OPEN_YR_CORE_WHEEL_URL:-}" +override_sha256="${OPEN_YR_CORE_WHEEL_SHA256:-}" + +if [[ -n "${override_url}" && -z "${override_sha256}" ]] || \ + [[ -z "${override_url}" && -n "${override_sha256}" ]]; then + echo "OPEN_YR_CORE_WHEEL_URL and OPEN_YR_CORE_WHEEL_SHA256 must be set together" >&2 + exit 1 +fi + +if [[ -n "${override_url}" ]]; then + wheel_url="${override_url}" + wheel_sha256="${override_sha256}" + wheel_name="$(python3 -c ' +import os +import sys +import urllib.parse + +print(os.path.basename(urllib.parse.unquote(urllib.parse.urlparse(sys.argv[1]).path))) +' "${wheel_url}")" +else + case "${TARGETARCH:-}" in + amd64) + wheel_arch=x86_64 + wheel_sha256="${OPEN_YR_CORE_AMD64_SHA256:?OPEN_YR_CORE_AMD64_SHA256 is required}" + ;; + arm64) + wheel_arch=aarch64 + wheel_sha256="${OPEN_YR_CORE_ARM64_SHA256:?OPEN_YR_CORE_ARM64_SHA256 is required}" + ;; + "") + case "$(uname -m)" in + x86_64) + wheel_arch=x86_64 + wheel_sha256="${OPEN_YR_CORE_AMD64_SHA256:?OPEN_YR_CORE_AMD64_SHA256 is required}" + ;; + aarch64|arm64) + wheel_arch=aarch64 + wheel_sha256="${OPEN_YR_CORE_ARM64_SHA256:?OPEN_YR_CORE_ARM64_SHA256 is required}" + ;; + *) + echo "unsupported openYuanRong target architecture: $(uname -m)" >&2 + exit 1 + ;; + esac + ;; + *) + echo "unsupported openYuanRong target architecture: ${TARGETARCH}" >&2 + exit 1 + ;; + esac + + open_yr_version="${OPEN_YR_VERSION:?OPEN_YR_VERSION is required}" + release_base_url="${OPEN_YR_RELEASE_BASE_URL:?OPEN_YR_RELEASE_BASE_URL is required}" + wheel_name="openyuanrong_core-${open_yr_version}-py3-none-manylinux_2_31_${wheel_arch}.whl" + wheel_url="${release_base_url}/${open_yr_version}/${wheel_name}" +fi + +case "${wheel_name}" in + ?*.whl) ;; + *) + echo "openYuanRong core URL must reference a .whl file: ${wheel_url}" >&2 + exit 1 + ;; +esac + +mkdir -p "${destination_dir}" +destination="${destination_dir}/${wheel_name}" +[[ ! -e "${destination}" ]] || { + echo "openYuanRong core destination already exists: ${destination}" >&2 + exit 1 +} + +temporary_dir="$(mktemp -d)" +trap 'rm -rf "${temporary_dir}"' EXIT +temporary_wheel="${temporary_dir}/${wheel_name}" + +curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ + "${wheel_url}" -o "${temporary_wheel}" +echo "${wheel_sha256} ${temporary_wheel}" | sha256sum -c - +mv "${temporary_wheel}" "${destination}" diff --git a/builder/downloaders/download-openyuanrong-rrt.sh b/builder/downloaders/download-openyuanrong-rrt.sh new file mode 100755 index 0000000..5d29889 --- /dev/null +++ b/builder/downloaders/download-openyuanrong-rrt.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ "$#" -ne 1 ]]; then + echo "usage: $0 DEST_FILE" >&2 + exit 2 +fi + +destination="$1" +override_url="${OPEN_YR_RRT_WHEEL_URL:-}" +override_sha256="${OPEN_YR_RRT_WHEEL_SHA256:-}" + +if [[ -n "${override_url}" && -z "${override_sha256}" ]] || \ + [[ -z "${override_url}" && -n "${override_sha256}" ]]; then + echo "OPEN_YR_RRT_WHEEL_URL and OPEN_YR_RRT_WHEEL_SHA256 must be set together" >&2 + exit 1 +fi + +destination_dir="$(dirname "${destination}")" +mkdir -p "${destination_dir}" +[[ ! -d "${destination}" ]] || { + echo "RRT destination must be a file path: ${destination}" >&2 + exit 1 +} + +temporary_dir="$(mktemp -d "${destination_dir}/.openyuanrong-rrt.XXXXXX")" +trap 'rm -rf "${temporary_dir}"' EXIT +candidate="${temporary_dir}/rrt-runtime" + +if [[ -n "${override_url}" ]]; then + wheel="${temporary_dir}/openyuanrong-rrt.whl" + curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ + "${override_url}" -o "${wheel}" + echo "${override_sha256} ${wheel}" | sha256sum -c - + unzip -p "${wheel}" openyuanrong_rrt/rrt-runtime > "${candidate}" +else + runtime_url="${RRT_RUNTIME_URL:?RRT_RUNTIME_URL is required}" + runtime_sha256="${RRT_RUNTIME_SHA256:?RRT_RUNTIME_SHA256 is required}" + curl -fSL --retry 5 --retry-delay 2 --retry-all-errors \ + "${runtime_url}" -o "${candidate}" + echo "${runtime_sha256} ${candidate}" | sha256sum -c - +fi + +[[ -s "${candidate}" ]] || { + echo "downloaded RRT runtime is empty" >&2 + exit 1 +} +mv "${candidate}" "${destination}" diff --git a/builder/downloaders/tests/fixtures/curl b/builder/downloaders/tests/fixtures/curl new file mode 100755 index 0000000..153b820 --- /dev/null +++ b/builder/downloaders/tests/fixtures/curl @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -euo pipefail + +destination="" +while [[ "$#" -gt 0 ]]; do + case "$1" in + -o) + destination="$2" + shift 2 + ;; + *) + shift + ;; + esac +done + +[[ -n "${destination}" ]] +mkdir -p "${FAKE_CURL_CALL_DIR}" +: >"${FAKE_CURL_CALL_DIR}/call-$$-${RANDOM}" +if [[ -n "${FAKE_CURL_DELAY:-}" ]]; then + sleep "${FAKE_CURL_DELAY}" +fi +if [[ "${FAKE_CURL_FAIL:-0}" == "1" ]]; then + printf 'partial\n' >"${destination}" + exit 28 +fi +cp "${FAKE_CURL_SOURCE}" "${destination}" diff --git a/builder/downloaders/tests/test-cache-verified-download.sh b/builder/downloaders/tests/test-cache-verified-download.sh new file mode 100755 index 0000000..5e0ac8e --- /dev/null +++ b/builder/downloaders/tests/test-cache-verified-download.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +DOWNLOADER="${ROOT}/builder/downloaders/cache-verified-download.sh" + +test_root="$(mktemp -d)" +trap 'rm -rf "${test_root}"' EXIT + +fixture="${test_root}/fixture.tar.zst" +printf 'verified archive fixture\n' >"${fixture}" +expected_sha="$(sha256sum "${fixture}" | awk '{print $1}')" +cache_file="${test_root}/cache/kata/4.0.0/amd64/${expected_sha}/archive.tar.zst" +call_dir="${test_root}/calls" + +call_count() { + if [[ ! -d "${call_dir}" ]]; then + printf '0\n' + return + fi + find "${call_dir}" -type f -name 'call-*' | wc -l | tr -d ' ' +} + +export PATH="${ROOT}/builder/downloaders/tests/fixtures:${PATH}" +export FAKE_CURL_SOURCE="${fixture}" +export FAKE_CURL_CALL_DIR="${call_dir}" + +first_output="$("${DOWNLOADER}" \ + https://example.invalid/archive "${expected_sha}" "${cache_file}")" +[[ "${first_output}" == "cache-fill ${cache_file}" ]] +cmp "${fixture}" "${cache_file}" +[[ "$(call_count)" == "1" ]] + +second_output="$("${DOWNLOADER}" \ + https://example.invalid/archive "${expected_sha}" "${cache_file}")" +[[ "${second_output}" == "cache-hit ${cache_file}" ]] +[[ "$(call_count)" == "1" ]] + +chmod u+w "${cache_file}" +printf 'corrupt\n' >"${cache_file}" +replacement_output="$("${DOWNLOADER}" \ + https://example.invalid/archive "${expected_sha}" "${cache_file}")" +[[ "${replacement_output}" == "cache-fill ${cache_file}" ]] +cmp "${fixture}" "${cache_file}" +[[ "$(call_count)" == "2" ]] + +expected_sha512="$(sha512sum "${fixture}" | awk '{print $1}')" +sha512_file="${test_root}/cache/gvisor/release-test/x86_64/${expected_sha512}/runsc" +sha512_output="$("${DOWNLOADER}" \ + https://example.invalid/runsc "${expected_sha512}" "${sha512_file}")" +[[ "${sha512_output}" == "cache-fill ${sha512_file}" ]] +cmp "${fixture}" "${sha512_file}" +[[ "$(call_count)" == "3" ]] + +invalid_digest="$(printf '%096d' 0 | tr 0 a)" +invalid_call_count="$(call_count)" +if "${DOWNLOADER}" https://example.invalid/invalid "${invalid_digest}" \ + "${test_root}/cache/invalid/runsc" >"${test_root}/invalid.log" 2>&1; then + echo "invalid digest unexpectedly succeeded" >&2 + exit 1 +fi +[[ "$(call_count)" == "${invalid_call_count}" ]] + +mismatch_file="${test_root}/cache/mismatch/archive.tar.zst" +bad_source="${test_root}/bad-source.tar.zst" +printf 'wrong bytes\n' >"${bad_source}" +if FAKE_CURL_SOURCE="${bad_source}" "${DOWNLOADER}" \ + https://example.invalid/archive "${expected_sha}" "${mismatch_file}" \ + >"${test_root}/mismatch.log" 2>&1; then + echo "checksum mismatch unexpectedly succeeded" >&2 + exit 1 +fi +[[ ! -e "${mismatch_file}" ]] +if find "$(dirname "${mismatch_file}")" -name '*.part.*' -print -quit | grep -q .; then + echo "checksum mismatch left a partial cache file" >&2 + exit 1 +fi + +interrupted_file="${test_root}/cache/interrupted/archive.tar.zst" +if FAKE_CURL_FAIL=1 "${DOWNLOADER}" \ + https://example.invalid/archive "${expected_sha}" "${interrupted_file}"; then + echo "interrupted download unexpectedly succeeded" >&2 + exit 1 +fi +[[ ! -e "${interrupted_file}" ]] +if find "$(dirname "${interrupted_file}")" -name '*.part.*' -print -quit | grep -q .; then + echo "interrupted download left a partial cache file" >&2 + exit 1 +fi + +concurrent_calls="${test_root}/concurrent-calls" +concurrent_file="${test_root}/cache/concurrent/archive.tar.zst" +FAKE_CURL_CALL_DIR="${concurrent_calls}" FAKE_CURL_DELAY=0.2 \ + "${DOWNLOADER}" https://example.invalid/archive \ + "${expected_sha}" "${concurrent_file}" >"${test_root}/concurrent-1.log" & +first_pid=$! +FAKE_CURL_CALL_DIR="${concurrent_calls}" FAKE_CURL_DELAY=0.2 \ + "${DOWNLOADER}" https://example.invalid/archive \ + "${expected_sha}" "${concurrent_file}" >"${test_root}/concurrent-2.log" & +second_pid=$! +wait "${first_pid}" +wait "${second_pid}" +cmp "${fixture}" "${concurrent_file}" +[[ "$(find "${concurrent_calls}" -type f -name 'call-*' | wc -l | tr -d ' ')" == "2" ]] +if find "$(dirname "${concurrent_file}")" -name '*.part.*' -print -quit | grep -q .; then + echo "concurrent downloads left a partial cache file" >&2 + exit 1 +fi + +echo "cache downloader checks passed" diff --git a/builder/downloaders/tests/test-openyuanrong-downloaders.sh b/builder/downloaders/tests/test-openyuanrong-downloaders.sh new file mode 100755 index 0000000..775e67f --- /dev/null +++ b/builder/downloaders/tests/test-openyuanrong-downloaders.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +CORE_DOWNLOADER="${ROOT}/builder/downloaders/download-openyuanrong-core.sh" +RRT_DOWNLOADER="${ROOT}/builder/downloaders/download-openyuanrong-rrt.sh" +TMP="$(mktemp -d)" +trap 'rm -rf "${TMP}"' EXIT + +sha256() { + sha256sum "$1" | awk '{print $1}' +} + +assert_file_equals() { + local expected="$1" + local actual="$2" + + cmp -s "${expected}" "${actual}" || { + echo "file mismatch: expected ${expected}, got ${actual}" >&2 + exit 1 + } +} + +assert_directory_empty() { + local directory="$1" + + if find "${directory}" -mindepth 1 -print -quit | grep -q .; then + echo "expected empty directory after failed download: ${directory}" >&2 + exit 1 + fi +} + +release_root="${TMP}/release" +core_release_dir="${release_root}/0.9.7" +core_release_name="openyuanrong_core-0.9.7-py3-none-manylinux_2_31_x86_64.whl" +mkdir -p "${core_release_dir}" +printf 'release-core-wheel\n' > "${core_release_dir}/${core_release_name}" +core_release_sha="$(sha256 "${core_release_dir}/${core_release_name}")" + +core_release_output="${TMP}/core-release-output" +OPEN_YR_VERSION=0.9.7 \ +OPEN_YR_RELEASE_BASE_URL="file://${release_root}" \ +OPEN_YR_CORE_AMD64_SHA256="${core_release_sha}" \ +OPEN_YR_CORE_ARM64_SHA256=unused \ +OPEN_YR_CORE_WHEEL_URL='' \ +OPEN_YR_CORE_WHEEL_SHA256='' \ +TARGETARCH=amd64 \ + "${CORE_DOWNLOADER}" "${core_release_output}" +assert_file_equals \ + "${core_release_dir}/${core_release_name}" \ + "${core_release_output}/${core_release_name}" + +obs_core_name="openyuanrong_core-0.7.0+build221-py3-none-manylinux_2_31_x86_64.whl" +obs_core="${TMP}/${obs_core_name}" +printf 'obs-core-wheel\n' > "${obs_core}" +obs_core_sha="$(sha256 "${obs_core}")" +obs_core_url="file://${obs_core}" +obs_core_url="${obs_core_url/+/%2B}" +core_obs_output="${TMP}/core-obs-output" +OPEN_YR_VERSION=ignored \ +OPEN_YR_RELEASE_BASE_URL=ignored \ +OPEN_YR_CORE_AMD64_SHA256=unused \ +OPEN_YR_CORE_ARM64_SHA256=unused \ +OPEN_YR_CORE_WHEEL_URL="${obs_core_url}" \ +OPEN_YR_CORE_WHEEL_SHA256="${obs_core_sha}" \ +TARGETARCH=amd64 \ + "${CORE_DOWNLOADER}" "${core_obs_output}" +assert_file_equals "${obs_core}" "${core_obs_output}/${obs_core_name}" + +core_bad_output="${TMP}/core-bad-output" +mkdir -p "${core_bad_output}" +if OPEN_YR_VERSION=ignored \ + OPEN_YR_RELEASE_BASE_URL=ignored \ + OPEN_YR_CORE_AMD64_SHA256=unused \ + OPEN_YR_CORE_ARM64_SHA256=unused \ + OPEN_YR_CORE_WHEEL_URL="${obs_core_url}" \ + OPEN_YR_CORE_WHEEL_SHA256=ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff \ + TARGETARCH=amd64 \ + "${CORE_DOWNLOADER}" "${core_bad_output}" >/dev/null 2>&1; then + echo "core downloader accepted an invalid checksum" >&2 + exit 1 +fi +assert_directory_empty "${core_bad_output}" + +core_unpaired_output="${TMP}/core-unpaired-output" +mkdir -p "${core_unpaired_output}" +if OPEN_YR_CORE_WHEEL_URL="${obs_core_url}" \ + OPEN_YR_CORE_WHEEL_SHA256='' \ + "${CORE_DOWNLOADER}" "${core_unpaired_output}" >/dev/null 2>&1; then + echo "core downloader accepted an unpaired URL" >&2 + exit 1 +fi +assert_directory_empty "${core_unpaired_output}" + +rrt_release="${TMP}/rrt-runtime-amd64" +printf 'release-rrt-runtime\n' > "${rrt_release}" +rrt_release_sha="$(sha256 "${rrt_release}")" +rrt_release_output="${TMP}/rrt-release-output" +RRT_RUNTIME_URL="file://${rrt_release}" \ +RRT_RUNTIME_SHA256="${rrt_release_sha}" \ +OPEN_YR_RRT_WHEEL_URL='' \ +OPEN_YR_RRT_WHEEL_SHA256='' \ + "${RRT_DOWNLOADER}" "${rrt_release_output}" +assert_file_equals "${rrt_release}" "${rrt_release_output}" + +rrt_wheel_root="${TMP}/rrt-wheel-root" +rrt_wheel_name="openyuanrong_rrt-0.7.0+build221-py3-none-manylinux_2_31_x86_64.whl" +rrt_wheel="${TMP}/${rrt_wheel_name}" +mkdir -p "${rrt_wheel_root}/openyuanrong_rrt" +printf 'obs-rrt-runtime\n' > "${rrt_wheel_root}/openyuanrong_rrt/rrt-runtime" +( + cd "${rrt_wheel_root}" + zip -q "${rrt_wheel}" openyuanrong_rrt/rrt-runtime +) +rrt_wheel_sha="$(sha256 "${rrt_wheel}")" +rrt_wheel_url="file://${rrt_wheel}" +rrt_wheel_url="${rrt_wheel_url/+/%2B}" +rrt_wheel_output="${TMP}/rrt-wheel-output" +OPEN_YR_RRT_WHEEL_URL="${rrt_wheel_url}" \ +OPEN_YR_RRT_WHEEL_SHA256="${rrt_wheel_sha}" \ + "${RRT_DOWNLOADER}" "${rrt_wheel_output}" +assert_file_equals \ + "${rrt_wheel_root}/openyuanrong_rrt/rrt-runtime" \ + "${rrt_wheel_output}" + +missing_member_root="${TMP}/missing-member-root" +missing_member_wheel="${TMP}/missing-member.whl" +mkdir -p "${missing_member_root}/openyuanrong_rrt" +printf 'metadata only\n' > "${missing_member_root}/openyuanrong_rrt/METADATA" +( + cd "${missing_member_root}" + zip -q "${missing_member_wheel}" openyuanrong_rrt/METADATA +) +missing_member_output="${TMP}/missing-member-output" +if OPEN_YR_RRT_WHEEL_URL="file://${missing_member_wheel}" \ + OPEN_YR_RRT_WHEEL_SHA256="$(sha256 "${missing_member_wheel}")" \ + "${RRT_DOWNLOADER}" "${missing_member_output}" >/dev/null 2>&1; then + echo "RRT downloader accepted a wheel without rrt-runtime" >&2 + exit 1 +fi +[[ ! -e "${missing_member_output}" ]] || { + echo "RRT downloader published output for a missing wheel member" >&2 + exit 1 +} + +rrt_unpaired_output="${TMP}/rrt-unpaired-output" +if OPEN_YR_RRT_WHEEL_URL="${rrt_wheel_url}" \ + OPEN_YR_RRT_WHEEL_SHA256='' \ + "${RRT_DOWNLOADER}" "${rrt_unpaired_output}" >/dev/null 2>&1; then + echo "RRT downloader accepted an unpaired URL" >&2 + exit 1 +fi +[[ ! -e "${rrt_unpaired_output}" ]] || { + echo "RRT downloader published output for an unpaired URL" >&2 + exit 1 +} + +echo "openYuanRong downloader behavior checks passed" diff --git a/builder/node.Dockerfile b/builder/node.Dockerfile index 67ffb22..7bdb45b 100644 --- a/builder/node.Dockerfile +++ b/builder/node.Dockerfile @@ -14,6 +14,7 @@ ARG OPEN_YR_RELEASE_BASE_URL=https://github.com/openYuanrong-mirror/yuanrong/rel ARG OPEN_YR_CORE_AMD64_SHA256=0a890db1785e349bfd625844a05059bdd494e32a429cea771cf969f09e3aba2c ARG OPEN_YR_CORE_ARM64_SHA256=64e14233fcbbb3418311d2f242e164e7be6e7bee0315c7619b18d9c5ddd01a76 ARG GVISOR_RELEASE=release-20260706.0 +ARG GVISOR_AMD64_SHA512=73938c145ebe554cf61a01da455688f4b732eebdf7b1b635bdef5b195868b363d8cb400e3d92ed1f377b78996805556c247a4849583910cb04e92b156053033e ARG GVISOR_RELEASE_BASE_URL=https://storage.googleapis.com/gvisor/releases ARG LIBNVIDIA_CONTAINER_VERSION=1.19.1-1 ARG KATA_BUILD_IMAGE=ubuntu:24.04 @@ -21,24 +22,40 @@ ARG KATA_RELEASE=4.0.0 ARG KATA_AMD64_SHA256=2c3b9dfeba355582b40aee462b12916c9740654d0230f696adf719d67b063a8c ARG KATA_RELEASE_BASE_URL=https://github.com/kata-containers/kata-containers/releases/download ARG OTELCOL_CONTRIB_VERSION=0.120.0 +ARG OTELCOL_CONTRIB_SHA256=81bf885bc9a86705feb3c113c5a356571390e3601eb651ffcf2b3428f6571adb ARG OTELCOL_CONTRIB_URL=https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${OTELCOL_CONTRIB_VERSION}/otelcol-contrib_${OTELCOL_CONTRIB_VERSION}_linux_amd64.tar.gz ARG AKERNEL_VERSION=unknown ARG AKERNEL_REVISION=unknown +ARG AKERNEL_INCLUDE_KATA=true +ARG AKERNEL_INCLUDE_NVIDIA=true FROM ${KATA_BUILD_IMAGE} AS kata-runtime +ARG AKERNEL_INCLUDE_KATA ARG KATA_RELEASE ARG KATA_AMD64_SHA256 ARG KATA_RELEASE_BASE_URL ARG TARGETARCH -RUN set -eux; \ +RUN --mount=type=bind,from=akernel-download-cache,target=/var/cache/akernel-downloads,ro \ + set -eux; \ + case "${AKERNEL_INCLUDE_KATA}" in true|false) ;; *) exit 1 ;; esac; \ + mkdir -p /kata/opt/kata; \ + if [ "${AKERNEL_INCLUDE_KATA}" = "false" ]; then \ + exit 0; \ + fi; \ test "${TARGETARCH:-amd64}" = "amd64"; \ apt-get update; \ apt-get install -y --no-install-recommends ca-certificates curl zstd; \ rm -rf /var/lib/apt/lists/*; \ archive="/tmp/kata-static-${KATA_RELEASE}-amd64.tar.zst"; \ - curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ - "${KATA_RELEASE_BASE_URL}/${KATA_RELEASE}/kata-static-${KATA_RELEASE}-amd64.tar.zst" \ - -o "${archive}"; \ + cache_archive="/var/cache/akernel-downloads/kata/${KATA_RELEASE}/amd64/${KATA_AMD64_SHA256}/kata-static-${KATA_RELEASE}-amd64.tar.zst"; \ + if [ -f "${cache_archive}" ]; then \ + echo "kata-cache-hit ${cache_archive}"; \ + cp "${cache_archive}" "${archive}"; \ + else \ + curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ + "${KATA_RELEASE_BASE_URL}/${KATA_RELEASE}/kata-static-${KATA_RELEASE}-amd64.tar.zst" \ + -o "${archive}"; \ + fi; \ echo "${KATA_AMD64_SHA256} ${archive}" | sha256sum -c -; \ mkdir -p /kata; \ tar --zstd -xf "${archive}" -C /kata \ @@ -74,7 +91,8 @@ RUN make release FROM ${DISTILL_FS_BUILD_IMAGE} AS distill-fs-builder ENV DEBIAN_FRONTEND=noninteractive \ - CARGO_NET_GIT_FETCH_WITH_CLI=true + CARGO_NET_GIT_FETCH_WITH_CLI=true \ + CARGO_NET_RETRY=5 RUN apt-get update && \ apt-get install -y --no-install-recommends \ ca-certificates \ @@ -88,12 +106,25 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* WORKDIR /src/distill-fs COPY ./src/distill-fs/ ./ -RUN cargo build --locked --release --bin distill_fs +RUN set -eux; \ + for attempt in 1 2 3; do \ + if cargo build --locked --release --bin distill_fs; then \ + exit 0; \ + fi; \ + if [ "${attempt}" -eq 3 ]; then \ + exit 1; \ + fi; \ + rm -rf /usr/local/cargo/git/db/nydus-* \ + /usr/local/cargo/git/checkouts/nydus-*; \ + sleep "$((attempt * 2))"; \ + done FROM ${AKERNEL_NODE_BASE_IMAGE} ARG AKERNEL_RUNTIME_PROFILE ARG AKERNEL_VERSION ARG AKERNEL_REVISION +ARG AKERNEL_INCLUDE_KATA +ARG AKERNEL_INCLUDE_NVIDIA ARG OPEN_YR_VERSION ARG OPEN_YR_CORE_WHEEL_URL ARG OPEN_YR_CORE_WHEEL_SHA256 @@ -101,8 +132,11 @@ ARG OPEN_YR_RELEASE_BASE_URL ARG OPEN_YR_CORE_AMD64_SHA256 ARG OPEN_YR_CORE_ARM64_SHA256 ARG GVISOR_RELEASE +ARG GVISOR_AMD64_SHA512 ARG GVISOR_RELEASE_BASE_URL ARG LIBNVIDIA_CONTAINER_VERSION +ARG OTELCOL_CONTRIB_VERSION +ARG OTELCOL_CONTRIB_SHA256 ARG OTELCOL_CONTRIB_URL ARG TARGETARCH ARG PIP_INDEX_URL=https://pypi.org/simple @@ -123,6 +157,7 @@ RUN apt-get update && \ logrotate \ mount \ openssl \ + patch \ procps \ python3 \ python3-pip \ @@ -133,6 +168,10 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* RUN set -eux; \ + case "${AKERNEL_INCLUDE_NVIDIA}" in true|false) ;; *) exit 1 ;; esac; \ + if [ "${AKERNEL_INCLUDE_NVIDIA}" = "false" ]; then \ + exit 0; \ + fi; \ curl -fsSL --retry 10 --retry-delay 2 --retry-all-errors \ https://nvidia.github.io/libnvidia-container/gpgkey \ | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg; \ @@ -152,7 +191,8 @@ RUN if command -v update-alternatives >/dev/null 2>&1; then \ update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy || true; \ fi -RUN set -eux; \ +RUN --mount=type=bind,from=akernel-download-cache,target=/var/cache/akernel-downloads,ro \ + set -eux; \ case "${TARGETARCH:-}" in \ amd64) gvisor_arch="x86_64" ;; \ "") \ @@ -168,9 +208,15 @@ RUN set -eux; \ gvisor_url="${GVISOR_RELEASE_BASE_URL}/release/${gvisor_version}/${gvisor_arch}"; \ mkdir -p /tmp/gvisor-release; \ cd /tmp/gvisor-release; \ - curl -fSLO --retry 10 --retry-delay 2 --retry-all-errors "${gvisor_url}/runsc"; \ - curl -fSLO --retry 10 --retry-delay 2 --retry-all-errors "${gvisor_url}/runsc.sha512"; \ - sha512sum -c runsc.sha512; \ + cache_runsc="/var/cache/akernel-downloads/gvisor/${GVISOR_RELEASE}/x86_64/${GVISOR_AMD64_SHA512}/runsc"; \ + if [ -f "${cache_runsc}" ]; then \ + echo "gvisor-cache-hit ${cache_runsc}"; \ + cp "${cache_runsc}" runsc; \ + else \ + curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ + "${gvisor_url}/runsc" -o runsc; \ + fi; \ + echo "${GVISOR_AMD64_SHA512} runsc" | sha512sum -c -; \ install -m 0755 runsc /usr/local/bin/runsc; \ rm -rf /tmp/gvisor-release @@ -194,35 +240,17 @@ ENV YR_INSTALLATION_DIR=/home/yuanrong # Install the complete, language-runtime-free openYuanRong control plane from # its checksum-pinned core wheel. A URL and checksum pair may override the # release asset when validating an unreleased daily build. +COPY ./builder/downloaders/download-openyuanrong-core.sh /usr/local/bin/ RUN set -eux; \ - case "${TARGETARCH:-}" in \ - amd64) wheel_arch=x86_64; release_sha="${OPEN_YR_CORE_AMD64_SHA256}" ;; \ - arm64) wheel_arch=aarch64; release_sha="${OPEN_YR_CORE_ARM64_SHA256}" ;; \ - "") \ - case "$(uname -m)" in \ - x86_64) wheel_arch=x86_64; release_sha="${OPEN_YR_CORE_AMD64_SHA256}" ;; \ - aarch64) wheel_arch=aarch64; release_sha="${OPEN_YR_CORE_ARM64_SHA256}" ;; \ - *) echo "unsupported openYuanRong target architecture: $(uname -m)" >&2; exit 1 ;; \ - esac ;; \ - *) echo "unsupported openYuanRong target architecture: ${TARGETARCH}" >&2; exit 1 ;; \ - esac; \ - wheel_name="openyuanrong_core-${OPEN_YR_VERSION}-py3-none-manylinux_2_31_${wheel_arch}.whl"; \ - wheel_url="${OPEN_YR_RELEASE_BASE_URL}/${OPEN_YR_VERSION}/${wheel_name}"; \ - wheel_sha="${release_sha}"; \ - if [ -n "${OPEN_YR_CORE_WHEEL_URL}" ]; then \ - test -n "${OPEN_YR_CORE_WHEEL_SHA256}"; \ - wheel_name="$(python3 -c 'import os, sys, urllib.parse; print(os.path.basename(urllib.parse.unquote(urllib.parse.urlparse(sys.argv[1]).path)))' "${OPEN_YR_CORE_WHEEL_URL}")"; \ - case "${wheel_name}" in *.whl) ;; *) echo "OPEN_YR_CORE_WHEEL_URL must reference a .whl file" >&2; exit 1 ;; esac; \ - wheel_url="${OPEN_YR_CORE_WHEEL_URL}"; \ - wheel_sha="${OPEN_YR_CORE_WHEEL_SHA256}"; \ - else \ - test -z "${OPEN_YR_CORE_WHEEL_SHA256}"; \ - fi; \ - wheel="/tmp/${wheel_name}"; \ + download_dir=/tmp/openyuanrong-core-download; \ + mkdir -p "${download_dir}"; \ + chmod 0755 /usr/local/bin/download-openyuanrong-core.sh; \ + /usr/local/bin/download-openyuanrong-core.sh "${download_dir}"; \ + set -- "${download_dir}"/*.whl; \ + test "$#" -eq 1; \ + wheel="$1"; \ + test -f "${wheel}"; \ target=/tmp/openyuanrong-core; \ - curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ - "${wheel_url}" -o "${wheel}"; \ - echo "${wheel_sha} ${wheel}" | sha256sum -c -; \ python3 -m pip install \ --break-system-packages \ --no-cache-dir \ @@ -232,9 +260,23 @@ RUN set -eux; \ test -x "${target}/yr/functionsystem/bin/yr"; \ mkdir -p "${YR_INSTALLATION_DIR}"; \ cp -a "${target}/yr/." "${YR_INSTALLATION_DIR}/"; \ - rm -rf "${target}" "${wheel}"; \ + rm -rf "${target}" "${download_dir}"; \ + rm -f /usr/local/bin/download-openyuanrong-core.sh; \ ln -sfn "${YR_INSTALLATION_DIR}/functionsystem/bin/yr" /usr/bin/yr +COPY ./builder/patches/openyuanrong-core-6dfa49681774-pause-resume-process.patch /usr/local/patches/ +COPY ./builder/patches/openyuanrong-core-454473b64447-pause-resume-process.patch /usr/local/patches/ +COPY ./builder/patches/openyuanrong-core-obs-snapshot-process.patch /usr/local/patches/ +COPY ./builder/scripts/apply-openyuanrong-pause-resume-patch.sh /usr/local/bin/ +COPY ./builder/scripts/apply-openyuanrong-obs-snapshot-patch.sh /usr/local/bin/ +RUN set -eux; \ + chmod 0755 /usr/local/bin/apply-openyuanrong-pause-resume-patch.sh \ + /usr/local/bin/apply-openyuanrong-obs-snapshot-patch.sh; \ + if [ -n "${OPEN_YR_CORE_WHEEL_SHA256}" ]; then \ + /usr/local/bin/apply-openyuanrong-pause-resume-patch.sh \ + "${YR_INSTALLATION_DIR}" "${OPEN_YR_CORE_WHEEL_SHA256}"; \ + fi + COPY --from=runtime-image /yr-runtime-rootfs.img ${YR_INSTALLATION_DIR}/yr-runtime-rootfs.img COPY --from=sandboxd-builder /src/sandboxd/output/sandboxd /usr/local/bin/sandboxd @@ -242,7 +284,12 @@ COPY --from=sandboxd-builder /src/sandboxd/output/sbox /usr/local/bin/sbox COPY --from=sandboxd-builder /src/sandboxd/output/sandbox-logger /usr/local/bin/sandbox-logger COPY --from=distill-fs-builder /src/distill-fs/target/release/distill_fs /usr/local/bin/distill_fs COPY --from=kata-runtime /kata/opt/kata /opt/kata -RUN ln -sf /opt/kata/runtime-rs/bin/containerd-shim-kata-v2 /usr/local/bin/containerd-shim-kata-v2 +RUN set -eux; \ + case "${AKERNEL_INCLUDE_KATA}" in true|false) ;; *) exit 1 ;; esac; \ + if [ "${AKERNEL_INCLUDE_KATA}" = "true" ]; then \ + ln -sf /opt/kata/runtime-rs/bin/containerd-shim-kata-v2 \ + /usr/local/bin/containerd-shim-kata-v2; \ + fi COPY ./builder/scripts/akernel-entrypoint.sh /usr/local/bin/akernel-entrypoint COPY ./builder/scripts/ensure-component-cert.sh /usr/local/bin/ensure-component-cert @@ -253,10 +300,12 @@ RUN chmod 0755 \ /usr/local/bin/sbox \ /usr/local/bin/sandbox-logger \ /usr/local/bin/distill_fs \ - /usr/local/bin/containerd-shim-kata-v2 \ /usr/local/bin/akernel-entrypoint \ /usr/local/bin/ensure-component-cert \ - /usr/local/bin/sandboxd-network-prepare + /usr/local/bin/sandboxd-network-prepare && \ + if [ "${AKERNEL_INCLUDE_KATA}" = "true" ]; then \ + chmod 0755 /usr/local/bin/containerd-shim-kata-v2; \ + fi COPY ./builder/config/yr_services.yaml /tmp/yr_services_rrt.yaml COPY ./builder/config/yr_services_python.yaml /tmp/yr_services_python.yaml @@ -266,6 +315,7 @@ RUN set -eux; \ python) services=/tmp/yr_services_python.yaml ;; \ *) echo "unsupported AKERNEL_RUNTIME_PROFILE: ${AKERNEL_RUNTIME_PROFILE}" >&2; exit 1 ;; \ esac; \ + touch ${YR_INSTALLATION_DIR}/.akernel-rrt-capable; \ install -D -m 0644 "${services}" ${YR_INSTALLATION_DIR}/deploy/process/services.yaml; \ rm -f /tmp/yr_services_rrt.yaml /tmp/yr_services_python.yaml @@ -279,10 +329,21 @@ COPY ./builder/scripts/master_entrypoint.sh ${YR_INSTALLATION_DIR}/entrypoint.sh COPY ./builder/scripts/*.sh /root/ COPY ./builder/systemd_services/*.service /etc/systemd/system/ -RUN curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ - "${OTELCOL_CONTRIB_URL}" \ - | tar -xz -C /usr/local/bin otelcol-contrib && \ - chmod 0755 /usr/local/bin/otelcol-contrib +RUN --mount=type=bind,from=akernel-download-cache,target=/var/cache/akernel-downloads,ro \ + set -eux; \ + archive="/tmp/otelcol-contrib_${OTELCOL_CONTRIB_VERSION}_linux_amd64.tar.gz"; \ + cache_archive="/var/cache/akernel-downloads/otelcol-contrib/${OTELCOL_CONTRIB_VERSION}/linux-amd64/${OTELCOL_CONTRIB_SHA256}/otelcol-contrib_${OTELCOL_CONTRIB_VERSION}_linux_amd64.tar.gz"; \ + if [ -f "${cache_archive}" ]; then \ + echo "otelcol-cache-hit ${cache_archive}"; \ + cp "${cache_archive}" "${archive}"; \ + else \ + curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ + "${OTELCOL_CONTRIB_URL}" -o "${archive}"; \ + fi; \ + echo "${OTELCOL_CONTRIB_SHA256} ${archive}" | sha256sum -c -; \ + tar -xzf "${archive}" -C /usr/local/bin otelcol-contrib; \ + chmod 0755 /usr/local/bin/otelcol-contrib; \ + rm -f "${archive}" RUN mkdir -p ${YR_INSTALLATION_DIR}/logs ${YR_INSTALLATION_DIR}/metrics ${YR_INSTALLATION_DIR}/trace && \ chmod 0755 ${YR_INSTALLATION_DIR}/yr_node_bootstrap.sh ${YR_INSTALLATION_DIR}/entrypoint.sh && \ diff --git a/builder/patches/openyuanrong-core-454473b64447-pause-resume-process.patch b/builder/patches/openyuanrong-core-454473b64447-pause-resume-process.patch new file mode 100644 index 0000000..1e6747d --- /dev/null +++ b/builder/patches/openyuanrong-core-454473b64447-pause-resume-process.patch @@ -0,0 +1,101 @@ +--- a/deploy/process/config.sh ++++ b/deploy/process/config.sh +@@ -88,6 +88,7 @@ + enable_traefik_registry:,enable_traefik_provider:,traefik_domain:,traefik_etcd_prefix:,traefik_lease_ttl:,traefik_http_entrypoint:,traefik_http_entry_point:,traefik_enable_tls:,traefik_servers_transport:,traefik_forward_timeout_ms:,\ + meta_service_address:,\ + system_tenant_id:,\ ++enable_sandbox_pause_resume:,snapshot_storage_backend:,checkpoint_dir:,\ + help" + FS_LOG_CONFIG="{\"filepath\": \"{{logConfigPath}}\",\"level\": \"{{logLevel}}\",\"compress\": {{logCompressEnable}}, \ + \"rolling\": {\"maxsize\": {{logRollingMaxSize}},\"maxfiles\": {{logRollingMaxFiles}},\"retentionDays\": {{logRollingRetentionDays}}}, \ +@@ -174,6 +175,9 @@ + RUNTIME_METRICS_CONFIG_FILE="" + YR_DATASYSTEM_DEFAULT_WRITE_MODE="NONE_L2_CACHE" + STATE_STORAGE_TYPE="datasystem" ++ENABLE_SANDBOX_PAUSE_RESUME="false" ++SNAPSHOT_STORAGE_BACKEND="" ++CHECKPOINT_DIR="" + PULL_RESOURCE_INTERVAL=1000 + BLOCK=false + ENABLE_MULTI_MASTER="false" +@@ -984,12 +988,29 @@ + --dposix_uds_path) DPOSIX_UDS_PATH=$2 && shift 2 ;; + --local_ip) LOCAL_IP=$2 && shift 2 ;; + --system_tenant_id) SYSTEM_TENANT_ID=$2 && shift 2 ;; ++ --enable_sandbox_pause_resume) ENABLE_SANDBOX_PAUSE_RESUME=$2 && shift 2 ;; ++ --snapshot_storage_backend) SNAPSHOT_STORAGE_BACKEND=$2 && shift 2 ;; ++ --checkpoint_dir) CHECKPOINT_DIR=$2 && shift 2 ;; + --) shift && break ;; + *) log_error "Invalid option: $1" && return 1 ;; + esac + done + if [ -z "${RUNTIME_METRICS_CONFIG_FILE}" ]; then + RUNTIME_METRICS_CONFIG_FILE="${METRICS_CONFIG_FILE}" ++ fi ++ if [ "X${ENABLE_SANDBOX_PAUSE_RESUME}" != "Xtrue" ] && [ "X${ENABLE_SANDBOX_PAUSE_RESUME}" != "Xfalse" ]; then ++ log_error "enable_sandbox_pause_resume can only be 'true' or 'false'" ++ return 1 ++ fi ++ if [ "X${ENABLE_SANDBOX_PAUSE_RESUME}" = "Xtrue" ]; then ++ if [ "X${SNAPSHOT_STORAGE_BACKEND}" != "Xdatasystem" ]; then ++ log_error "standalone pause/resume requires snapshot_storage_backend=datasystem" ++ return 1 ++ fi ++ if [ -z "${CHECKPOINT_DIR}" ] || [ "${CHECKPOINT_DIR#/}" = "${CHECKPOINT_DIR}" ]; then ++ log_error "checkpoint_dir must be an absolute path when pause/resume is enabled" ++ return 1 ++ fi + fi + } + +@@ -1815,6 +1836,7 @@ + export RUNTIME_INIT_PORT DS_WORKER_PORT RUNTIME_CONN_TIMEOUT_S + export ENABLE_RUNTIME_LAUNCHER RUNTIME_LAUNCHER_SOCK + export RUNTIME_INIT_CALL_TIMEOUT_SECONDS IS_SCHEDULE_TOLERATE_ABNORMAL STATE_STORAGE_TYPE ++ export ENABLE_SANDBOX_PAUSE_RESUME SNAPSHOT_STORAGE_BACKEND CHECKPOINT_DIR + export MERGE_PROCESS_ENABLE FUNCTION_PROXY_MERGE_PROCESS_ENABLE DATA_SYSTEM_ENABLE DRIVER_GATEWAY_ENABLE SSH_ENABLE ENABLE_TCP_TUNNEL + export FRONTEND_SSH_AUTH_ENABLE FRONTEND_SSH_ADDRESS FRONTEND_SSH_HOST_KEY FRONTEND_SSH_AUTHORIZED_KEYS + export FRONTEND_SSH_BACKEND_KEY FRONTEND_SSH_MAX_CONNECTIONS TCP_TUNNEL_PORT TCP_TUNNEL_MAX_CONNECTIONS +--- a/functionsystem/deploy/install.sh ++++ b/functionsystem/deploy/install.sh +@@ -63,7 +63,11 @@ + + function get_runtime_checkpoint_dir() { + local checkpoint_dir +- checkpoint_dir=$(readlink -m "${FUNCTION_SYSTEM_DIR}/../checkpoints") ++ if [ -n "${CHECKPOINT_DIR:-}" ]; then ++ checkpoint_dir=$(readlink -m "${CHECKPOINT_DIR}") ++ else ++ checkpoint_dir=$(readlink -m "${FUNCTION_SYSTEM_DIR}/../checkpoints") ++ fi + mkdir -p "${checkpoint_dir}" + echo "${checkpoint_dir}" + } +@@ -99,6 +103,8 @@ + + local merge_process_args="" + if [ "X${FUNCTION_PROXY_MERGE_PROCESS_ENABLE^^}" == "XTRUE" ]; then ++ local checkpoint_dir ++ checkpoint_dir=$(get_runtime_checkpoint_dir) + local ld_library_path=${LD_LIBRARY_PATH} + local function_system_ld_library_path=${FUNCTION_SYSTEM_DIR}/lib:${DATA_SYSTEM_DIR}/lib:${ld_library_path} + local agent_uid=${YR_POD_NAME} +@@ -106,6 +112,9 @@ + agent_uid="${NODE_ID}" + fi + merge_process_args="--enable_merge_process=true \ ++ --enable_sandbox_pause_resume="${ENABLE_SANDBOX_PAUSE_RESUME}" \ ++ --snapshot_storage_backend="${SNAPSHOT_STORAGE_BACKEND}" \ ++ --checkpoint_dir="${checkpoint_dir}" \ + --agent_listen_port="${FUNCTION_PROXY_PORT}" \ + --local_scheduler_address="${IP_ADDRESS}:${FUNCTION_PROXY_PORT}" \ + --runtime_dir="${RUNTIME_HOME_DIR}/service" \ +@@ -591,6 +600,8 @@ + # Extract agent arguments into an array for clarity and reuse + agent_args=( + --enable_merge_process=true ++ --enable_sandbox_pause_resume="${ENABLE_SANDBOX_PAUSE_RESUME}" ++ --snapshot_storage_backend="${SNAPSHOT_STORAGE_BACKEND}" + --ip="${IP_ADDRESS}" + --node_id="${NODE_ID}" + --agent_uid="${agent_uid}" diff --git a/builder/patches/openyuanrong-core-6dfa49681774-pause-resume-process.patch b/builder/patches/openyuanrong-core-6dfa49681774-pause-resume-process.patch new file mode 100644 index 0000000..3126279 --- /dev/null +++ b/builder/patches/openyuanrong-core-6dfa49681774-pause-resume-process.patch @@ -0,0 +1,101 @@ +--- a/deploy/process/config.sh ++++ b/deploy/process/config.sh +@@ -88,6 +88,7 @@ + enable_traefik_registry:,enable_traefik_provider:,traefik_domain:,traefik_etcd_prefix:,traefik_lease_ttl:,traefik_http_entrypoint:,traefik_http_entry_point:,traefik_enable_tls:,traefik_servers_transport:,traefik_forward_timeout_ms:,\ + meta_service_address:,\ + system_tenant_id:,\ ++enable_sandbox_pause_resume:,snapshot_storage_backend:,checkpoint_dir:,\ + help" + FS_LOG_CONFIG="{\"filepath\": \"{{logConfigPath}}\",\"level\": \"{{logLevel}}\",\"compress\": {{logCompressEnable}}, \ + \"rolling\": {\"maxsize\": {{logRollingMaxSize}},\"maxfiles\": {{logRollingMaxFiles}},\"retentionDays\": {{logRollingRetentionDays}}}, \ +@@ -174,6 +175,9 @@ + RUNTIME_METRICS_CONFIG_FILE="" + YR_DATASYSTEM_DEFAULT_WRITE_MODE="NONE_L2_CACHE" + STATE_STORAGE_TYPE="datasystem" ++ENABLE_SANDBOX_PAUSE_RESUME="false" ++SNAPSHOT_STORAGE_BACKEND="" ++CHECKPOINT_DIR="" + PULL_RESOURCE_INTERVAL=1000 + BLOCK=false + ENABLE_MULTI_MASTER="false" +@@ -981,12 +985,29 @@ + --dposix_uds_path) DPOSIX_UDS_PATH=$2 && shift 2 ;; + --local_ip) LOCAL_IP=$2 && shift 2 ;; + --system_tenant_id) SYSTEM_TENANT_ID=$2 && shift 2 ;; ++ --enable_sandbox_pause_resume) ENABLE_SANDBOX_PAUSE_RESUME=$2 && shift 2 ;; ++ --snapshot_storage_backend) SNAPSHOT_STORAGE_BACKEND=$2 && shift 2 ;; ++ --checkpoint_dir) CHECKPOINT_DIR=$2 && shift 2 ;; + --) shift && break ;; + *) log_error "Invalid option: $1" && return 1 ;; + esac + done + if [ -z "${RUNTIME_METRICS_CONFIG_FILE}" ]; then + RUNTIME_METRICS_CONFIG_FILE="${METRICS_CONFIG_FILE}" ++ fi ++ if [ "X${ENABLE_SANDBOX_PAUSE_RESUME}" != "Xtrue" ] && [ "X${ENABLE_SANDBOX_PAUSE_RESUME}" != "Xfalse" ]; then ++ log_error "enable_sandbox_pause_resume can only be 'true' or 'false'" ++ return 1 ++ fi ++ if [ "X${ENABLE_SANDBOX_PAUSE_RESUME}" = "Xtrue" ]; then ++ if [ "X${SNAPSHOT_STORAGE_BACKEND}" != "Xdatasystem" ]; then ++ log_error "standalone pause/resume requires snapshot_storage_backend=datasystem" ++ return 1 ++ fi ++ if [ -z "${CHECKPOINT_DIR}" ] || [ "${CHECKPOINT_DIR#/}" = "${CHECKPOINT_DIR}" ]; then ++ log_error "checkpoint_dir must be an absolute path when pause/resume is enabled" ++ return 1 ++ fi + fi + } + +@@ -1808,6 +1829,7 @@ + export RUNTIME_INIT_PORT DS_WORKER_PORT RUNTIME_CONN_TIMEOUT_S + export ENABLE_RUNTIME_LAUNCHER RUNTIME_LAUNCHER_SOCK + export RUNTIME_INIT_CALL_TIMEOUT_SECONDS IS_SCHEDULE_TOLERATE_ABNORMAL STATE_STORAGE_TYPE ++ export ENABLE_SANDBOX_PAUSE_RESUME SNAPSHOT_STORAGE_BACKEND CHECKPOINT_DIR + export MERGE_PROCESS_ENABLE FUNCTION_PROXY_MERGE_PROCESS_ENABLE DRIVER_GATEWAY_ENABLE SSH_ENABLE ENABLE_TCP_TUNNEL + export FRONTEND_SSH_AUTH_ENABLE FRONTEND_SSH_ADDRESS FRONTEND_SSH_HOST_KEY FRONTEND_SSH_AUTHORIZED_KEYS + export FRONTEND_SSH_BACKEND_KEY FRONTEND_SSH_MAX_CONNECTIONS TCP_TUNNEL_PORT TCP_TUNNEL_MAX_CONNECTIONS +--- a/functionsystem/deploy/install.sh ++++ b/functionsystem/deploy/install.sh +@@ -63,7 +63,11 @@ + + function get_runtime_checkpoint_dir() { + local checkpoint_dir +- checkpoint_dir=$(readlink -m "${FUNCTION_SYSTEM_DIR}/../checkpoints") ++ if [ -n "${CHECKPOINT_DIR:-}" ]; then ++ checkpoint_dir=$(readlink -m "${CHECKPOINT_DIR}") ++ else ++ checkpoint_dir=$(readlink -m "${FUNCTION_SYSTEM_DIR}/../checkpoints") ++ fi + mkdir -p "${checkpoint_dir}" + echo "${checkpoint_dir}" + } +@@ -99,6 +103,8 @@ + + local merge_process_args="" + if [ "X${FUNCTION_PROXY_MERGE_PROCESS_ENABLE^^}" == "XTRUE" ]; then ++ local checkpoint_dir ++ checkpoint_dir=$(get_runtime_checkpoint_dir) + local ld_library_path=${LD_LIBRARY_PATH} + local function_system_ld_library_path=${FUNCTION_SYSTEM_DIR}/lib:${DATA_SYSTEM_DIR}/lib:${ld_library_path} + local agent_uid=${YR_POD_NAME} +@@ -106,6 +112,9 @@ + agent_uid="${NODE_ID}" + fi + merge_process_args="--enable_merge_process=true \ ++ --enable_sandbox_pause_resume="${ENABLE_SANDBOX_PAUSE_RESUME}" \ ++ --snapshot_storage_backend="${SNAPSHOT_STORAGE_BACKEND}" \ ++ --checkpoint_dir="${checkpoint_dir}" \ + --agent_listen_port="${FUNCTION_PROXY_PORT}" \ + --local_scheduler_address="${IP_ADDRESS}:${FUNCTION_PROXY_PORT}" \ + --runtime_dir="${RUNTIME_HOME_DIR}/service" \ +@@ -591,6 +600,8 @@ + # Extract agent arguments into an array for clarity and reuse + agent_args=( + --enable_merge_process=true ++ --enable_sandbox_pause_resume="${ENABLE_SANDBOX_PAUSE_RESUME}" ++ --snapshot_storage_backend="${SNAPSHOT_STORAGE_BACKEND}" + --ip="${IP_ADDRESS}" + --node_id="${NODE_ID}" + --agent_uid="${agent_uid}" diff --git a/builder/patches/openyuanrong-core-obs-snapshot-process.patch b/builder/patches/openyuanrong-core-obs-snapshot-process.patch new file mode 100644 index 0000000..1199810 --- /dev/null +++ b/builder/patches/openyuanrong-core-obs-snapshot-process.patch @@ -0,0 +1,82 @@ +--- a/deploy/process/config.sh ++++ b/deploy/process/config.sh +@@ -88,7 +88,8 @@ + enable_traefik_registry:,enable_traefik_provider:,traefik_domain:,traefik_etcd_prefix:,traefik_lease_ttl:,traefik_http_entrypoint:,traefik_http_entry_point:,traefik_enable_tls:,traefik_servers_transport:,traefik_forward_timeout_ms:,\ + meta_service_address:,\ + system_tenant_id:,\ +-enable_sandbox_pause_resume:,snapshot_storage_backend:,checkpoint_dir:,\ ++enable_sandbox_pause_resume:,snapshot_storage_backend:,checkpoint_dir:,snapshot_obs_endpoint:,snapshot_obs_bucket:,\ ++snapshot_obs_access_key:,snapshot_obs_secret_key:,snapshot_obs_security_token:,snapshot_obs_use_https:,snapshot_obs_path_style:,\ + help" + FS_LOG_CONFIG="{\"filepath\": \"{{logConfigPath}}\",\"level\": \"{{logLevel}}\",\"compress\": {{logCompressEnable}}, \ + \"rolling\": {\"maxsize\": {{logRollingMaxSize}},\"maxfiles\": {{logRollingMaxFiles}},\"retentionDays\": {{logRollingRetentionDays}}}, \ +@@ -178,6 +179,13 @@ + ENABLE_SANDBOX_PAUSE_RESUME="false" + SNAPSHOT_STORAGE_BACKEND="" + CHECKPOINT_DIR="" ++SNAPSHOT_OBS_ENDPOINT="" ++SNAPSHOT_OBS_BUCKET="" ++SNAPSHOT_OBS_ACCESS_KEY="" ++SNAPSHOT_OBS_SECRET_KEY="" ++SNAPSHOT_OBS_SECURITY_TOKEN="" ++SNAPSHOT_OBS_USE_HTTPS="true" ++SNAPSHOT_OBS_PATH_STYLE="false" + PULL_RESOURCE_INTERVAL=1000 + BLOCK=false + ENABLE_MULTI_MASTER="false" +@@ -991,6 +999,13 @@ + --enable_sandbox_pause_resume) ENABLE_SANDBOX_PAUSE_RESUME=$2 && shift 2 ;; + --snapshot_storage_backend) SNAPSHOT_STORAGE_BACKEND=$2 && shift 2 ;; + --checkpoint_dir) CHECKPOINT_DIR=$2 && shift 2 ;; ++ --snapshot_obs_endpoint) SNAPSHOT_OBS_ENDPOINT=$2 && shift 2 ;; ++ --snapshot_obs_bucket) SNAPSHOT_OBS_BUCKET=$2 && shift 2 ;; ++ --snapshot_obs_access_key) SNAPSHOT_OBS_ACCESS_KEY=$2 && shift 2 ;; ++ --snapshot_obs_secret_key) SNAPSHOT_OBS_SECRET_KEY=$2 && shift 2 ;; ++ --snapshot_obs_security_token) SNAPSHOT_OBS_SECURITY_TOKEN=$2 && shift 2 ;; ++ --snapshot_obs_use_https) SNAPSHOT_OBS_USE_HTTPS=$2 && shift 2 ;; ++ --snapshot_obs_path_style) SNAPSHOT_OBS_PATH_STYLE=$2 && shift 2 ;; + --) shift && break ;; + *) log_error "Invalid option: $1" && return 1 ;; + esac +@@ -1003,10 +1018,28 @@ + return 1 + fi + if [ "X${ENABLE_SANDBOX_PAUSE_RESUME}" = "Xtrue" ]; then +- if [ "X${SNAPSHOT_STORAGE_BACKEND}" != "Xdatasystem" ]; then +- log_error "standalone pause/resume requires snapshot_storage_backend=datasystem" +- return 1 +- fi ++ case "${SNAPSHOT_STORAGE_BACKEND}" in ++ datasystem) ;; ++ obs) ++ if [ -z "${SNAPSHOT_OBS_ENDPOINT}" ] || [ -z "${SNAPSHOT_OBS_BUCKET}" ] || \ ++ [ -z "${SNAPSHOT_OBS_ACCESS_KEY}" ] || [ -z "${SNAPSHOT_OBS_SECRET_KEY}" ]; then ++ log_error "OBS snapshot storage requires endpoint, bucket, access key, and secret key" ++ return 1 ++ fi ++ case "${SNAPSHOT_OBS_USE_HTTPS}" in ++ true|false) ;; ++ *) log_error "snapshot_obs_use_https can only be 'true' or 'false'" && return 1 ;; ++ esac ++ case "${SNAPSHOT_OBS_PATH_STYLE}" in ++ true|false) ;; ++ *) log_error "snapshot_obs_path_style can only be 'true' or 'false'" && return 1 ;; ++ esac ++ ;; ++ *) ++ log_error "snapshot_storage_backend must be datasystem or obs" ++ return 1 ++ ;; ++ esac + if [ -z "${CHECKPOINT_DIR}" ] || [ "${CHECKPOINT_DIR#/}" = "${CHECKPOINT_DIR}" ]; then + log_error "checkpoint_dir must be an absolute path when pause/resume is enabled" + return 1 +@@ -1837,6 +1870,8 @@ + export ENABLE_RUNTIME_LAUNCHER RUNTIME_LAUNCHER_SOCK + export RUNTIME_INIT_CALL_TIMEOUT_SECONDS IS_SCHEDULE_TOLERATE_ABNORMAL STATE_STORAGE_TYPE + export ENABLE_SANDBOX_PAUSE_RESUME SNAPSHOT_STORAGE_BACKEND CHECKPOINT_DIR ++ export SNAPSHOT_OBS_ENDPOINT SNAPSHOT_OBS_BUCKET SNAPSHOT_OBS_ACCESS_KEY SNAPSHOT_OBS_SECRET_KEY ++ export SNAPSHOT_OBS_SECURITY_TOKEN SNAPSHOT_OBS_USE_HTTPS SNAPSHOT_OBS_PATH_STYLE + export MERGE_PROCESS_ENABLE FUNCTION_PROXY_MERGE_PROCESS_ENABLE DATA_SYSTEM_ENABLE DRIVER_GATEWAY_ENABLE SSH_ENABLE ENABLE_TCP_TUNNEL + export FRONTEND_SSH_AUTH_ENABLE FRONTEND_SSH_ADDRESS FRONTEND_SSH_HOST_KEY FRONTEND_SSH_AUTHORIZED_KEYS + export FRONTEND_SSH_BACKEND_KEY FRONTEND_SSH_MAX_CONNECTIONS TCP_TUNNEL_PORT TCP_TUNNEL_MAX_CONNECTIONS diff --git a/builder/runtime.Dockerfile b/builder/runtime.Dockerfile index 953e9c1..469afc8 100644 --- a/builder/runtime.Dockerfile +++ b/builder/runtime.Dockerfile @@ -10,22 +10,29 @@ ARG PYTHON_312_VERSION=3.12.11 ARG PYTHON_313_VERSION=3.13.5 ARG PYTHON_314_VERSION=3.14.6 ARG OPEN_YR_VERSION=0.9.7 +ARG OPEN_YR_RRT_WHEEL_URL= +ARG OPEN_YR_RRT_WHEEL_SHA256= FROM ${AKERNEL_RUNTIME_BASE_IMAGE} AS rrt-download ARG OPEN_YR_VERSION ARG RRT_RUNTIME_URL=https://github.com/openYuanrong-mirror/yuanrong/releases/download/${OPEN_YR_VERSION}/rrt-runtime-amd64 ARG RRT_RUNTIME_SHA256=c7f7441ed631de564f56c1878b84598a7364ffae040a0e51f6e14a82f6ae31d5 +ARG OPEN_YR_RRT_WHEEL_URL +ARG OPEN_YR_RRT_WHEEL_SHA256 +ARG TARGETARCH RUN apt-get update && \ - apt-get install -y --no-install-recommends ca-certificates curl && \ + apt-get install -y --no-install-recommends ca-certificates curl file unzip && \ rm -rf /var/lib/apt/lists/* +COPY ./builder/downloaders/download-openyuanrong-rrt.sh /usr/local/bin/ RUN set -eux; \ - curl -fSL --retry 5 --retry-delay 2 --retry-all-errors \ - -o /rrt-runtime "${RRT_RUNTIME_URL}"; \ - echo "${RRT_RUNTIME_SHA256} /rrt-runtime" | sha256sum -c -; \ - chmod 0755 /rrt-runtime + case "${TARGETARCH:-amd64}" in amd64) ;; *) echo "RRT runtime only supports amd64" >&2; exit 1 ;; esac; \ + chmod 0755 /usr/local/bin/download-openyuanrong-rrt.sh; \ + /usr/local/bin/download-openyuanrong-rrt.sh /rrt-runtime; \ + chmod 0755 /rrt-runtime; \ + file /rrt-runtime | grep -Eq 'ELF 64-bit LSB.*x86-64' FROM ${AKERNEL_RUNTIME_BASE_IMAGE} AS rrt-runtime-rootfs @@ -111,12 +118,37 @@ RUN python3 -m pip install \ --retries 10 \ "uv==${UV_VERSION}" -RUN uv python install \ - "${PYTHON_310_VERSION}" \ - "${PYTHON_311_VERSION}" \ - "${PYTHON_312_VERSION}" \ - "${PYTHON_313_VERSION}" \ - "${PYTHON_314_VERSION}"; \ +ARG UV_PYTHON_INSTALL_MIRROR= +RUN set -eux; \ + attempt=1; \ + while true; do \ + if [ -n "${UV_PYTHON_INSTALL_MIRROR}" ]; then \ + if uv python install \ + --mirror "${UV_PYTHON_INSTALL_MIRROR}" \ + "${PYTHON_310_VERSION}" \ + "${PYTHON_311_VERSION}" \ + "${PYTHON_312_VERSION}" \ + "${PYTHON_313_VERSION}" \ + "${PYTHON_314_VERSION}"; then \ + break; \ + fi; \ + else \ + if uv python install \ + "${PYTHON_310_VERSION}" \ + "${PYTHON_311_VERSION}" \ + "${PYTHON_312_VERSION}" \ + "${PYTHON_313_VERSION}" \ + "${PYTHON_314_VERSION}"; then \ + break; \ + fi; \ + fi; \ + if [ "${attempt}" -ge 3 ]; then \ + echo "uv python install failed after 3 attempts" >&2; \ + exit 1; \ + fi; \ + sleep $((attempt * 5)); \ + attempt=$((attempt + 1)); \ + done; \ rm -rf "${UV_CACHE_DIR}" RUN set -eux; \ @@ -128,7 +160,8 @@ RUN set -eux; \ "3.14:${PYTHON_314_VERSION}"; do \ py="${spec%%:*}"; \ version="${spec#*:}"; \ - uv venv "/opt/venv-py${py}" --python "${version}" --seed; \ + UV_DEFAULT_INDEX="${PIP_INDEX_URL}" uv venv \ + "/opt/venv-py${py}" --python "${version}" --seed; \ ln -sfn \ "uv-python/cpython-${version}-linux-x86_64-gnu/bin/python${py}" \ "/opt/python${py}"; \ diff --git a/builder/scripts/apply-openyuanrong-obs-snapshot-patch.sh b/builder/scripts/apply-openyuanrong-obs-snapshot-patch.sh new file mode 100755 index 0000000..dba20d0 --- /dev/null +++ b/builder/scripts/apply-openyuanrong-obs-snapshot-patch.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -euo pipefail + +yr_root="${1:?usage: $0 YR_ROOT}" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +patch_file="${script_dir}/../patches/openyuanrong-core-obs-snapshot-process.patch" +config="${yr_root}/deploy/process/config.sh" +install="${yr_root}/functionsystem/deploy/install.sh" + +for required in "${config}" "${install}" "${patch_file}"; do + [[ -f "${required}" ]] || { + echo "missing ${required}" >&2 + exit 1 + } +done + +patch --directory="${yr_root}" --strip=1 --forward --batch --dry-run < "${patch_file}" >/dev/null +patch --directory="${yr_root}" --strip=1 --forward --batch < "${patch_file}" >/dev/null + +grep -Fq 'snapshot_obs_endpoint:' "${config}" +grep -Fq -- '--snapshot_obs_endpoint)' "${config}" +grep -Fq 'datasystem) ;;' "${config}" +grep -Fq ' obs)' "${config}" +grep -Fq 'export SNAPSHOT_OBS_ENDPOINT SNAPSHOT_OBS_BUCKET SNAPSHOT_OBS_ACCESS_KEY SNAPSHOT_OBS_SECRET_KEY' \ + "${config}" +grep -Fq 'export SNAPSHOT_OBS_SECURITY_TOKEN SNAPSHOT_OBS_USE_HTTPS SNAPSHOT_OBS_PATH_STYLE' \ + "${config}" +grep -Fq -- '--snapshot_obs_access_key="${SNAPSHOT_OBS_ACCESS_KEY:-}"' "${install}" +grep -Fq -- '--snapshot_obs_secret_key="${SNAPSHOT_OBS_SECRET_KEY:-}"' "${install}" diff --git a/builder/scripts/apply-openyuanrong-pause-resume-patch.sh b/builder/scripts/apply-openyuanrong-pause-resume-patch.sh new file mode 100755 index 0000000..1f1edfe --- /dev/null +++ b/builder/scripts/apply-openyuanrong-pause-resume-patch.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +set -euo pipefail + +yr_root="${1:?usage: $0 YR_ROOT CORE_SHA256}" +core_sha="${2:?usage: $0 YR_ROOT CORE_SHA256}" +mode="${3:-auto}" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +case "${mode}" in + auto|require|off) ;; + *) echo "pause/resume patch mode must be auto, require, or off" >&2; exit 2 ;; +esac +if [[ "${mode}" == "off" ]]; then + exit 0 +fi + +process_contract_present() { + local config="${yr_root}/deploy/process/config.sh" + local install="${yr_root}/functionsystem/deploy/install.sh" + [[ -f "${config}" && -f "${install}" ]] || return 1 + grep -Fq 'enable_sandbox_pause_resume:' "${config}" && + grep -Fq 'snapshot_storage_backend:' "${config}" && + grep -Fq 'checkpoint_dir:' "${config}" && + grep -Fq 'export ENABLE_SANDBOX_PAUSE_RESUME SNAPSHOT_STORAGE_BACKEND CHECKPOINT_DIR' \ + "${config}" && + grep -Fq -- '--enable_sandbox_pause_resume="${ENABLE_SANDBOX_PAUSE_RESUME}"' \ + "${install}" && + grep -Fq -- '--snapshot_storage_backend="${SNAPSHOT_STORAGE_BACKEND}"' \ + "${install}" && + grep -Fq -- '--checkpoint_dir="${checkpoint_dir}"' "${install}" +} + +case "${core_sha}" in + 39ba1cf8323ac4e784117867ecd806ec392da05aa2fd87130f8830eb56310895) + patch_file="${script_dir}/../patches/openyuanrong-core-6dfa49681774-pause-resume-process.patch" + ;; + 60d8af4fa5d46fae315461574f9f6653694e7327137f4bc9979633d31e5c6811) + patch_file="${script_dir}/../patches/openyuanrong-core-454473b64447-pause-resume-process.patch" + ;; + *) + if process_contract_present; then + echo "openYuanRong core ${core_sha} already provides the pause/resume process contract" + exit 0 + fi + if [[ "${mode}" == "auto" ]]; then + echo "openYuanRong core ${core_sha} is not a known legacy patch target; leaving it unchanged" >&2 + exit 0 + fi + echo "pause/resume process patch does not support openYuanRong core ${core_sha}" >&2 + exit 1 + ;; +esac + +for relative in deploy/process/config.sh functionsystem/deploy/install.sh; do + [[ -f "${yr_root}/${relative}" ]] || { + echo "missing ${yr_root}/${relative}" >&2 + exit 1 + } +done + +patch --directory="${yr_root}" --strip=1 --forward --batch --dry-run < "${patch_file}" >/dev/null +patch --directory="${yr_root}" --strip=1 --forward --batch < "${patch_file}" >/dev/null + +"${script_dir}/apply-openyuanrong-obs-snapshot-patch.sh" "${yr_root}" + +grep -Fq 'export ENABLE_SANDBOX_PAUSE_RESUME SNAPSHOT_STORAGE_BACKEND CHECKPOINT_DIR' \ + "${yr_root}/deploy/process/config.sh" +grep -Fq -- '--snapshot_storage_backend="${SNAPSHOT_STORAGE_BACKEND}"' \ + "${yr_root}/functionsystem/deploy/install.sh" diff --git a/builder/scripts/tests/test-openyuanrong-pause-resume-patch.sh b/builder/scripts/tests/test-openyuanrong-pause-resume-patch.sh new file mode 100755 index 0000000..7ddf1a7 --- /dev/null +++ b/builder/scripts/tests/test-openyuanrong-pause-resume-patch.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +CORE_URL='https://openyuanrong.obs.cn-southwest-2.myhuaweicloud.com/daily/20260812055037/linux/amd64/openyuanrong_core-0.7.0%2B454473b64447-py3-none-manylinux_2_31_x86_64.whl' +CORE_SHA='60d8af4fa5d46fae315461574f9f6653694e7327137f4bc9979633d31e5c6811' +TMP="$(mktemp -d)" +trap 'rm -rf "${TMP}"' EXIT + +if [[ -n "${OPEN_YR_PATCHED_TEST_ROOT:-}" ]]; then + mkdir -p "${TMP}/root" + cp -a "${OPEN_YR_PATCHED_TEST_ROOT}/yr" "${TMP}/root/yr" + "${ROOT}/builder/scripts/apply-openyuanrong-obs-snapshot-patch.sh" "${TMP}/root/yr" +else + wheel="${OPEN_YR_CORE_TEST_WHEEL:-${TMP}/core.whl}" + wheel_sha="${OPEN_YR_CORE_TEST_WHEEL_SHA256:-${CORE_SHA}}" + if [[ ! -f "${wheel}" ]]; then + curl -fSL --retry 5 --retry-delay 2 "${CORE_URL}" -o "${wheel}" + fi + echo "${wheel_sha} ${wheel}" | shasum -a 256 -c - >/dev/null + + python3 - "${wheel}" "${TMP}/root" <<'PY' +import pathlib +import sys +import zipfile + +wheel = pathlib.Path(sys.argv[1]) +root = pathlib.Path(sys.argv[2]) +members = ( + "yr/deploy/process/config.sh", + "yr/functionsystem/deploy/install.sh", +) +with zipfile.ZipFile(wheel) as archive: + for member in members: + archive.extract(member, root) +PY + + "${ROOT}/builder/scripts/apply-openyuanrong-pause-resume-patch.sh" "${TMP}/root/yr" "${CORE_SHA}" +fi + +config="${TMP}/root/yr/deploy/process/config.sh" +install="${TMP}/root/yr/functionsystem/deploy/install.sh" + +grep -Fq 'enable_sandbox_pause_resume:' "${config}" +grep -Fq 'snapshot_storage_backend:' "${config}" +grep -Fq 'checkpoint_dir:' "${config}" +for option in \ + snapshot_obs_endpoint snapshot_obs_bucket snapshot_obs_access_key \ + snapshot_obs_secret_key snapshot_obs_security_token \ + snapshot_obs_use_https snapshot_obs_path_style; do + grep -Fq "${option}:" "${config}" + grep -Fq -- "--${option})" "${config}" +done +grep -Fq 'data_system_enable:' "${config}" +grep -Fq -- '--enable_sandbox_pause_resume)' "${config}" +grep -Fq -- '--snapshot_storage_backend)' "${config}" +grep -Fq -- '--checkpoint_dir)' "${config}" +grep -Fq 'datasystem) ;;' "${config}" +grep -Fq ' obs)' "${config}" +grep -Fq 'export ENABLE_SANDBOX_PAUSE_RESUME SNAPSHOT_STORAGE_BACKEND CHECKPOINT_DIR' "${config}" +grep -Fq 'export SNAPSHOT_OBS_ENDPOINT SNAPSHOT_OBS_BUCKET SNAPSHOT_OBS_ACCESS_KEY SNAPSHOT_OBS_SECRET_KEY' "${config}" +grep -Fq 'export SNAPSHOT_OBS_SECURITY_TOKEN SNAPSHOT_OBS_USE_HTTPS SNAPSHOT_OBS_PATH_STYLE' "${config}" + +grep -Fq 'CHECKPOINT_DIR' "${install}" +grep -Fq -- '--data_system_enable="${DATA_SYSTEM_ENABLE:-false}"' "${install}" +grep -Fq -- '--enable_sandbox_pause_resume="${ENABLE_SANDBOX_PAUSE_RESUME:-false}"' "${install}" +grep -Fq -- '--snapshot_storage_backend="${SNAPSHOT_STORAGE_BACKEND:-datasystem}"' "${install}" +grep -Fq -- '--checkpoint_dir="${checkpoint_dir}"' "${install}" +for variable in endpoint bucket access_key secret_key security_token use_https path_style; do + case "${variable}" in + endpoint) expected='${SNAPSHOT_OBS_ENDPOINT:-}' ;; + bucket) expected='${SNAPSHOT_OBS_BUCKET:-}' ;; + access_key) expected='${SNAPSHOT_OBS_ACCESS_KEY:-}' ;; + secret_key) expected='${SNAPSHOT_OBS_SECRET_KEY:-}' ;; + security_token) expected='${SNAPSHOT_OBS_SECURITY_TOKEN:-}' ;; + use_https) expected='${SNAPSHOT_OBS_USE_HTTPS:-true}' ;; + path_style) expected='${SNAPSHOT_OBS_PATH_STYLE:-false}' ;; + esac + grep -Fq -- "--snapshot_obs_${variable}=\"${expected}\"" "${install}" +done + +count="$(grep -Fc -- '--snapshot_storage_backend="${SNAPSHOT_STORAGE_BACKEND:-datasystem}"' "${install}")" +[[ "${count}" -eq 2 ]] || { + echo "expected snapshot backend in merged proxy and standalone agent, got ${count}" >&2 + exit 1 +} + +for variable in \ + endpoint bucket access_key secret_key security_token use_https path_style; do + case "${variable}" in + endpoint) expected='${SNAPSHOT_OBS_ENDPOINT:-}' ;; + bucket) expected='${SNAPSHOT_OBS_BUCKET:-}' ;; + access_key) expected='${SNAPSHOT_OBS_ACCESS_KEY:-}' ;; + secret_key) expected='${SNAPSHOT_OBS_SECRET_KEY:-}' ;; + security_token) expected='${SNAPSHOT_OBS_SECURITY_TOKEN:-}' ;; + use_https) expected='${SNAPSHOT_OBS_USE_HTTPS:-true}' ;; + path_style) expected='${SNAPSHOT_OBS_PATH_STYLE:-false}' ;; + esac + count="$(grep -Fc -- "--snapshot_obs_${variable}=\"${expected}\"" "${install}")" + [[ "${count}" -eq 2 ]] || { + echo "expected snapshot OBS ${variable} in merged proxy and standalone agent, got ${count}" >&2 + exit 1 + } +done + +if [[ -n "${OPEN_YR_PATCHED_TEST_ROOT:-}" ]]; then + reapply=("${ROOT}/builder/scripts/apply-openyuanrong-obs-snapshot-patch.sh" "${TMP}/root/yr") +else + reapply=("${ROOT}/builder/scripts/apply-openyuanrong-pause-resume-patch.sh" "${TMP}/root/yr" "${CORE_SHA}") +fi +if "${reapply[@]}" >/dev/null 2>&1; then + echo "patch unexpectedly applied twice" >&2 + exit 1 +fi + +cp -a "${TMP}/root/yr" "${TMP}/already-patched" +"${ROOT}/builder/scripts/apply-openyuanrong-pause-resume-patch.sh" \ + "${TMP}/already-patched" "$(printf 'f%.0s' {1..64})" auto + +if [[ -z "${OPEN_YR_PATCHED_TEST_ROOT:-}" ]]; then + python3 - "${wheel}" "${TMP}/unknown" <<'PY' +import pathlib +import sys +import zipfile + +wheel = pathlib.Path(sys.argv[1]) +root = pathlib.Path(sys.argv[2]) +members = ( + "yr/deploy/process/config.sh", + "yr/functionsystem/deploy/install.sh", +) +with zipfile.ZipFile(wheel) as archive: + for member in members: + archive.extract(member, root) +PY + unknown_sha="$(printf 'e%.0s' {1..64})" + "${ROOT}/builder/scripts/apply-openyuanrong-pause-resume-patch.sh" \ + "${TMP}/unknown/yr" "${unknown_sha}" auto + if grep -Fq 'export ENABLE_SANDBOX_PAUSE_RESUME SNAPSHOT_STORAGE_BACKEND CHECKPOINT_DIR' \ + "${TMP}/unknown/yr/deploy/process/config.sh"; then + echo "auto mode unexpectedly modified an unknown core package" >&2 + exit 1 + fi + if "${ROOT}/builder/scripts/apply-openyuanrong-pause-resume-patch.sh" \ + "${TMP}/unknown/yr" "${unknown_sha}" require >/dev/null 2>&1; then + echo "require mode unexpectedly accepted an unsupported core package" >&2 + exit 1 + fi +fi + +echo "openYuanRong pause/resume process patch checks passed" diff --git a/builder/scripts/yr_node_bootstrap.sh b/builder/scripts/yr_node_bootstrap.sh index 52e5e21..0080726 100755 --- a/builder/scripts/yr_node_bootstrap.sh +++ b/builder/scripts/yr_node_bootstrap.sh @@ -5,6 +5,7 @@ # SPDX-License-Identifier: Apache-2.0 ulimit -n 32768 export YR_RUNTIME_BACKEND=sandboxd +source /root/yr_pause_resume_args.sh resolve_node_ip() { local default_device @@ -42,6 +43,13 @@ resolve_node_ip() { YR_NODE_IP="$(resolve_node_ip)" echo "Using ${YR_NODE_IP} as the YuanRong node address" +checkpoint_dir=/home/akernel/sandboxd/root/checkpoints +configure_pause_resume_args \ + "${AKERNEL_ENABLE_PAUSE_RESUME:-false}" \ + /home/yuanrong/.akernel-rrt-capable \ + "${checkpoint_dir}" \ + "${AKS_LOCAL_MODE:-false}" + # Select the legacy etcd registry or the FunctionMaster HTTP provider. if [ "${TRAEFIK_MODE:-etcd}" = "etcd" ]; then ENABLE_TRAEFIK_REGISTRY=${ENABLE_TRAEFIK_REGISTRY:-true} @@ -108,7 +116,8 @@ if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then --frontend_lease_bypass true \ --force_low_reliability_instance true \ --enable_sandbox_router true \ - --enable_direct_routing false + --enable_direct_routing false \ + "${standalone_pause_resume_args[@]}" else /usr/bin/yr start \ --ip_address "${YR_NODE_IP}" \ @@ -145,5 +154,6 @@ else --function_proxy_merge_process_enable true \ --enable_direct_routing false \ --force_low_reliability_instance true \ + "${pause_resume_args[@]}" \ --block true fi diff --git a/builder/scripts/yr_pause_resume_args.sh b/builder/scripts/yr_pause_resume_args.sh new file mode 100644 index 0000000..18ea5e4 --- /dev/null +++ b/builder/scripts/yr_pause_resume_args.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +# Copyright (c) 2026 Ant Group Corporation. +# +# SPDX-License-Identifier: Apache-2.0 + +configure_pause_resume_args() { + local enabled="${1:?pause/resume enabled value is required}" + local capability_file="${2:?RRT capability file is required}" + local checkpoint_dir="${3:?checkpoint directory is required}" + local standalone="${4:?standalone mode value is required}" + local snapshot_backend="${AKERNEL_SNAPSHOT_STORAGE_BACKEND:-datasystem}" + + pause_resume_args=() + standalone_pause_resume_args=() + + case "${enabled}" in + true) + if [ ! -f "${capability_file}" ]; then + echo "pause/resume requires an image built with the RRT runtime" >&2 + return 1 + fi + mkdir -p "${checkpoint_dir}" + if [ ! -w "${checkpoint_dir}" ]; then + echo "checkpoint directory is not writable: ${checkpoint_dir}" >&2 + return 1 + fi + case "${snapshot_backend}" in + datasystem) + pause_resume_args=( + --enable_sandbox_pause_resume true + --snapshot_storage_backend datasystem + --checkpoint_dir "${checkpoint_dir}" + ) + ;; + obs) + local obs_endpoint="${AKERNEL_SNAPSHOT_OBS_ENDPOINT:-}" + local obs_bucket="${AKERNEL_SNAPSHOT_OBS_BUCKET:-}" + local obs_access_key="${AKERNEL_SNAPSHOT_OBS_ACCESS_KEY:-}" + local obs_secret_key="${AKERNEL_SNAPSHOT_OBS_SECRET_KEY:-}" + local obs_security_token="${AKERNEL_SNAPSHOT_OBS_SECURITY_TOKEN:-}" + local obs_use_https="${AKERNEL_SNAPSHOT_OBS_USE_HTTPS:-true}" + local obs_path_style="${AKERNEL_SNAPSHOT_OBS_PATH_STYLE:-false}" + if [ -z "${obs_endpoint}" ] || [ -z "${obs_bucket}" ] || \ + [ -z "${obs_access_key}" ] || [ -z "${obs_secret_key}" ]; then + echo "OBS snapshot storage requires endpoint, bucket, access key, and secret key" >&2 + return 1 + fi + case "${obs_use_https}" in true|false) ;; *) + echo "AKERNEL_SNAPSHOT_OBS_USE_HTTPS must be true or false" >&2 + return 1 + esac + case "${obs_path_style}" in true|false) ;; *) + echo "AKERNEL_SNAPSHOT_OBS_PATH_STYLE must be true or false" >&2 + return 1 + esac + pause_resume_args=( + --enable_sandbox_pause_resume true + --snapshot_storage_backend obs + --checkpoint_dir "${checkpoint_dir}" + --snapshot_obs_endpoint "${obs_endpoint}" + --snapshot_obs_bucket "${obs_bucket}" + --snapshot_obs_access_key "${obs_access_key}" + --snapshot_obs_secret_key "${obs_secret_key}" + ) + if [ -n "${obs_security_token}" ]; then + pause_resume_args+=(--snapshot_obs_security_token "${obs_security_token}") + fi + pause_resume_args+=( + --snapshot_obs_use_https "${obs_use_https}" + --snapshot_obs_path_style "${obs_path_style}" + ) + ;; + *) + echo "AKERNEL_SNAPSHOT_STORAGE_BACKEND must be datasystem or obs" >&2 + return 1 + ;; + esac + standalone_pause_resume_args=("${pause_resume_args[@]}") + case "${standalone}" in + true) + standalone_pause_resume_args+=(--data_system_enable true) + ;; + false) ;; + *) + echo "AKS_LOCAL_MODE must be true or false" >&2 + return 1 + ;; + esac + ;; + false) ;; + *) + echo "AKERNEL_ENABLE_PAUSE_RESUME must be true or false" >&2 + return 1 + ;; + esac +} diff --git a/builder/systemd_services/yuanrong.service b/builder/systemd_services/yuanrong.service index 3960526..abc9ada 100644 --- a/builder/systemd_services/yuanrong.service +++ b/builder/systemd_services/yuanrong.service @@ -4,7 +4,8 @@ Description=yuanrong.service [Service] #Type=simple PIDFile=/run/yuanrong.pid -PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT +PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT AKERNEL_ENABLE_PAUSE_RESUME +PassEnvironment=AKERNEL_SNAPSHOT_STORAGE_BACKEND AKERNEL_SNAPSHOT_OBS_ENDPOINT AKERNEL_SNAPSHOT_OBS_BUCKET AKERNEL_SNAPSHOT_OBS_ACCESS_KEY AKERNEL_SNAPSHOT_OBS_SECRET_KEY AKERNEL_SNAPSHOT_OBS_SECURITY_TOKEN AKERNEL_SNAPSHOT_OBS_USE_HTTPS AKERNEL_SNAPSHOT_OBS_PATH_STYLE Environment="CONTAINER_EP=unix:///run/sandboxd/sandboxd.sock" Environment="RUNTIME_HOME_DIR=/home/yuanrong/runtime" Environment="YR_NOSET_CUDA_VISIBLE_DEVICES=1" diff --git a/deploy/README.md b/deploy/README.md index e36f430..0e10768 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -177,6 +177,12 @@ fsync enabled and uses persistent storage by default. Production environments that require etcd high availability should point AKernel at an externally managed multi-member etcd cluster instead of increasing `etcd.replicas`. +Kubernetes pause/resume is opt-in and requires an image built with the RRT +runtime profile. Enable it with `core.pauseResume.enabled=true`. The node +process then uses its DataSystem worker only as the checkpoint snapshot +backend; ordinary sandbox data-plane behavior is unchanged. The node fails +closed when the selected image lacks the RRT capability marker. + The core chart defaults master, frontend, and node to the same all-in-one image: ```yaml @@ -188,9 +194,25 @@ image: Each component can still override `master.image`, `frontend.image`, or `node.image` when a split-image deployment is required. -### Public Traefik entrypoints +### Frontend API and SandboxRouter endpoints + +The frontend Service exposes two distinct ports. The control API uses TLS on +8888, while public sandbox ports go directly to SandboxRouter over HTTP on +8080. Configure both addresses explicitly when the Service is reachable by a +client: + +```bash +export AKERNEL_SERVER_ADDRESS=https://:8888 +export AKERNEL_SANDBOX_ROUTER_ADDRESS=http://:8080 +``` + +The second endpoint serves `/{sandbox-id}/{container-port}/...` from the +authoritative SandboxRouter cache/read-through path. Platform JWTs are not +forwarded to user processes. Pause/resume route publication does not depend on +Traefik. -For cloud deployments, use Traefik with two public entrypoints: +Traefik remains available for legacy cloud profiles that need a single public +load balancer with two entrypoints: ```yaml traefik: @@ -202,9 +224,9 @@ traefik: ``` The `websecure` entrypoint serves the AKernel frontend API and exec websocket -over HTTPS/WSS. The `web` entrypoint serves function port-forwarding traffic -over plain HTTP/WS. With this layout the Python SDK only needs the LoadBalancer -host or IP: +over HTTPS/WSS. The `web` entrypoint forwards sandbox port traffic to +SandboxRouter over plain HTTP/WS. With this compatibility layout the Python SDK +only needs the LoadBalancer host or IP: ```bash export AKERNEL_SERVER_ADDRESS= diff --git a/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml b/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml index 2b02967..457cd90 100644 --- a/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml +++ b/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml @@ -110,6 +110,8 @@ spec: value: {{ get $frontendEtcd "host" | default (printf "akernel-etcd.%s.svc.cluster.local" .Release.Namespace) | quote }} - name: ETCD_PORT value: {{ get $frontendEtcd "port" | default "2379" | quote }} + - name: SANDBOX_ROUTER_PORT + value: {{ .Values.frontend.service.sandboxRouterPort | default 8080 | quote }} {{- if .Values.monitoring.prometheusEndpoint }} - name: PROMETHEUS_ENDPOINT value: {{ .Values.monitoring.prometheusEndpoint | quote }} @@ -131,6 +133,8 @@ spec: ports: - name: http containerPort: 8888 + - name: sandbox-router + containerPort: {{ .Values.frontend.service.sandboxRouterPort | default 8080 }} livenessProbe: httpGet: path: /healthz diff --git a/deploy/akernel/charts/core/templates/frontend/akernel_frontend_service.yaml b/deploy/akernel/charts/core/templates/frontend/akernel_frontend_service.yaml index 427c159..3ff3a25 100644 --- a/deploy/akernel/charts/core/templates/frontend/akernel_frontend_service.yaml +++ b/deploy/akernel/charts/core/templates/frontend/akernel_frontend_service.yaml @@ -23,6 +23,10 @@ spec: port: 8888 targetPort: 8888 protocol: TCP + - name: sandbox-router + port: {{ .Values.frontend.service.sandboxRouterPort | default 8080 }} + targetPort: {{ .Values.frontend.service.sandboxRouterPort | default 8080 }} + protocol: TCP selector: app: akernel-frontend sessionAffinity: None diff --git a/deploy/akernel/charts/core/templates/node/daemonset.yaml b/deploy/akernel/charts/core/templates/node/daemonset.yaml index 36b8443..6fe966d 100644 --- a/deploy/akernel/charts/core/templates/node/daemonset.yaml +++ b/deploy/akernel/charts/core/templates/node/daemonset.yaml @@ -80,6 +80,42 @@ spec: value: "node" - name: RUNSC_AKERNEL value: "1" + - name: AKERNEL_ENABLE_PAUSE_RESUME + value: {{ .Values.pauseResume.enabled | default false | quote }} + {{- $snapshotStorage := .Values.pauseResume.snapshotStorage | default dict }} + {{- $snapshotBackend := get $snapshotStorage "backend" | default "datasystem" }} + - name: AKERNEL_SNAPSHOT_STORAGE_BACKEND + value: {{ $snapshotBackend | quote }} + {{- if eq $snapshotBackend "obs" }} + {{- $snapshotObs := get $snapshotStorage "obs" | default dict }} + {{- $snapshotObsEndpoint := required "pauseResume.snapshotStorage.obs.endpoint is required for the OBS backend" (get $snapshotObs "endpoint") }} + {{- $snapshotObsBucket := required "pauseResume.snapshotStorage.obs.bucket is required for the OBS backend" (get $snapshotObs "bucket") }} + {{- $snapshotObsSecret := required "pauseResume.snapshotStorage.obs.existingSecret is required for the OBS backend" (get $snapshotObs "existingSecret") }} + - name: AKERNEL_SNAPSHOT_OBS_ENDPOINT + value: {{ $snapshotObsEndpoint | quote }} + - name: AKERNEL_SNAPSHOT_OBS_BUCKET + value: {{ $snapshotObsBucket | quote }} + - name: AKERNEL_SNAPSHOT_OBS_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ $snapshotObsSecret | quote }} + key: {{ get $snapshotObs "accessKeyKey" | default "access-key" | quote }} + - name: AKERNEL_SNAPSHOT_OBS_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ $snapshotObsSecret | quote }} + key: {{ get $snapshotObs "secretKeyKey" | default "secret-key" | quote }} + - name: AKERNEL_SNAPSHOT_OBS_SECURITY_TOKEN + valueFrom: + secretKeyRef: + name: {{ $snapshotObsSecret | quote }} + key: {{ get $snapshotObs "securityTokenKey" | default "security-token" | quote }} + optional: true + - name: AKERNEL_SNAPSHOT_OBS_USE_HTTPS + value: {{ get $snapshotObs "useHttps" | quote }} + - name: AKERNEL_SNAPSHOT_OBS_PATH_STYLE + value: {{ get $snapshotObs "pathStyle" | quote }} + {{- end }} - name: ETCD_ADDRESS value: {{ get $nodeEtcd "host" | default (printf "akernel-etcd.%s.svc.cluster.local" .Release.Namespace) | quote }} - name: ETCD_PORT diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index 5774525..b088b71 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -23,6 +23,26 @@ image: tag: "latest" pullPolicy: IfNotPresent +pauseResume: + # Requires an image built with the RRT runtime profile. The node process + # fails closed when this is enabled on an incompatible image. + enabled: false + snapshotStorage: + # DataSystem preserves the existing standalone and single-node behavior. + # Select obs when snapshots must survive termination of their source Pod. + backend: datasystem + obs: + endpoint: "" + bucket: "" + # This Secret must already exist in the release namespace. The chart + # never accepts or renders credential values directly. + existingSecret: "" + accessKeyKey: access-key + secretKeyKey: secret-key + securityTokenKey: security-token + useHttps: true + pathStyle: false + auth: # Existing Secret that contains the JWT signing seed. For # `helm template | kubectl apply`, prefer pre-creating this Secret with @@ -100,6 +120,7 @@ frontend: service: type: ClusterIP annotations: {} + sandboxRouterPort: 8080 # Image defaults to the global all-in-one image. Set these only to override. image: repository: "" diff --git a/deploy/akernel/tests/test-pause-resume-wiring.sh b/deploy/akernel/tests/test-pause-resume-wiring.sh new file mode 100644 index 0000000..ad451f8 --- /dev/null +++ b/deploy/akernel/tests/test-pause-resume-wiring.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +pause_resume_helper="${repo_root}/builder/scripts/yr_pause_resume_args.sh" +yuanrong_service="${repo_root}/builder/systemd_services/yuanrong.service" +tmp_dir="$(mktemp -d)" +cleanup() { + result=$? + rm -rf "${tmp_dir}" + exit "${result}" +} +trap cleanup EXIT + +# Exercise the production argument builder consumed by both launch modes. +if [[ ! -r "${pause_resume_helper}" ]]; then + echo "pause/resume argument builder is unavailable" >&2 + exit 1 +fi + +passed_environment=" $(sed -n 's/^PassEnvironment=//p' "${yuanrong_service}" | tr '\n' ' ') " +for variable in \ + AKERNEL_ENABLE_PAUSE_RESUME AKERNEL_SNAPSHOT_STORAGE_BACKEND \ + AKERNEL_SNAPSHOT_OBS_ENDPOINT AKERNEL_SNAPSHOT_OBS_BUCKET \ + AKERNEL_SNAPSHOT_OBS_ACCESS_KEY AKERNEL_SNAPSHOT_OBS_SECRET_KEY \ + AKERNEL_SNAPSHOT_OBS_SECURITY_TOKEN AKERNEL_SNAPSHOT_OBS_USE_HTTPS \ + AKERNEL_SNAPSHOT_OBS_PATH_STYLE; do + case "${passed_environment}" in + *" ${variable} "*) ;; + *) + echo "yuanrong.service does not pass ${variable}" >&2 + exit 1 + ;; + esac +done +source "${pause_resume_helper}" + +capability_file="${tmp_dir}/rrt-capable" +checkpoint_dir="${tmp_dir}/checkpoints" +touch "${capability_file}" + +configure_pause_resume_args \ + true "${capability_file}" "${checkpoint_dir}" false +[[ "${pause_resume_args[*]}" == \ + "--enable_sandbox_pause_resume true --snapshot_storage_backend datasystem --checkpoint_dir ${checkpoint_dir}" ]] +[[ "${standalone_pause_resume_args[*]}" == "${pause_resume_args[*]}" ]] +[[ -d "${checkpoint_dir}" ]] + +configure_pause_resume_args \ + true "${capability_file}" "${checkpoint_dir}" true +[[ "${standalone_pause_resume_args[*]}" == \ + "${pause_resume_args[*]} --data_system_enable true" ]] + +export AKERNEL_SNAPSHOT_STORAGE_BACKEND=obs +export AKERNEL_SNAPSHOT_OBS_ENDPOINT=obs.cn-north-4.myhuaweicloud.com +export AKERNEL_SNAPSHOT_OBS_BUCKET=akernel-test +export AKERNEL_SNAPSHOT_OBS_ACCESS_KEY=test-access-key +export AKERNEL_SNAPSHOT_OBS_SECRET_KEY=test-secret-key +export AKERNEL_SNAPSHOT_OBS_SECURITY_TOKEN=test-security-token +export AKERNEL_SNAPSHOT_OBS_USE_HTTPS=true +export AKERNEL_SNAPSHOT_OBS_PATH_STYLE=false +configure_pause_resume_args \ + true "${capability_file}" "${checkpoint_dir}" false +expected_obs_args="--enable_sandbox_pause_resume true --snapshot_storage_backend obs --checkpoint_dir ${checkpoint_dir} --snapshot_obs_endpoint obs.cn-north-4.myhuaweicloud.com --snapshot_obs_bucket akernel-test --snapshot_obs_access_key test-access-key --snapshot_obs_secret_key test-secret-key --snapshot_obs_security_token test-security-token --snapshot_obs_use_https true --snapshot_obs_path_style false" +if [[ "${pause_resume_args[*]}" != "${expected_obs_args}" ]]; then + echo "unexpected OBS pause/resume arguments" >&2 + exit 1 +fi + +unset AKERNEL_SNAPSHOT_OBS_SECRET_KEY +if configure_pause_resume_args \ + true "${capability_file}" "${checkpoint_dir}" false 2>/dev/null; then + echo "OBS snapshot storage accepted a missing secret key" >&2 + exit 1 +fi +unset AKERNEL_SNAPSHOT_STORAGE_BACKEND +unset AKERNEL_SNAPSHOT_OBS_ENDPOINT +unset AKERNEL_SNAPSHOT_OBS_BUCKET +unset AKERNEL_SNAPSHOT_OBS_ACCESS_KEY +unset AKERNEL_SNAPSHOT_OBS_SECURITY_TOKEN +unset AKERNEL_SNAPSHOT_OBS_USE_HTTPS +unset AKERNEL_SNAPSHOT_OBS_PATH_STYLE + +enabled_render="${tmp_dir}/enabled.yaml" +disabled_render="${tmp_dir}/disabled.yaml" +obs_render="${tmp_dir}/obs.yaml" +helm template akernel-wiring "${repo_root}/deploy/akernel" \ + --set monitor.enabled=false \ + --set core.pauseResume.enabled=true > "${enabled_render}" +helm template akernel-wiring "${repo_root}/deploy/akernel" \ + --set monitor.enabled=false > "${disabled_render}" +helm template akernel-wiring "${repo_root}/deploy/akernel" \ + --set monitor.enabled=false \ + --set core.pauseResume.enabled=true \ + --set core.pauseResume.snapshotStorage.backend=obs \ + --set core.pauseResume.snapshotStorage.obs.endpoint=obs.cn-north-4.myhuaweicloud.com \ + --set core.pauseResume.snapshotStorage.obs.bucket=akernel-test \ + --set core.pauseResume.snapshotStorage.obs.existingSecret=akernel-snapshot-obs \ + > "${obs_render}" + +python3 - "${enabled_render}" "${disabled_render}" "${obs_render}" <<'PY' +import sys + +import yaml + + +def node_environment(path: str) -> dict[str, dict]: + with open(path, encoding="utf-8") as stream: + documents = list(yaml.safe_load_all(stream)) + daemonset = next( + document + for document in documents + if document + and document.get("kind") == "DaemonSet" + and document["metadata"]["name"] == "akernel-node" + ) + env = daemonset["spec"]["template"]["spec"]["containers"][0]["env"] + return {item["name"]: item for item in env} + + +def frontend_documents(path: str): + with open(path, encoding="utf-8") as stream: + documents = list(yaml.safe_load_all(stream)) + deployment = next( + document + for document in documents + if document + and document.get("kind") == "Deployment" + and document["metadata"]["name"] == "akernel-frontend" + ) + service = next( + document + for document in documents + if document + and document.get("kind") == "Service" + and document["metadata"]["name"] == "akernel-frontend" + ) + return deployment, service + + +enabled = node_environment(sys.argv[1]) +disabled = node_environment(sys.argv[2]) +obs = node_environment(sys.argv[3]) + +assert enabled["AKERNEL_ENABLE_PAUSE_RESUME"]["value"] == "true" +assert disabled["AKERNEL_ENABLE_PAUSE_RESUME"]["value"] == "false" +assert enabled["AKERNEL_SNAPSHOT_STORAGE_BACKEND"]["value"] == "datasystem" + +assert obs["AKERNEL_SNAPSHOT_STORAGE_BACKEND"]["value"] == "obs" +assert obs["AKERNEL_SNAPSHOT_OBS_ENDPOINT"]["value"] == "obs.cn-north-4.myhuaweicloud.com" +assert obs["AKERNEL_SNAPSHOT_OBS_BUCKET"]["value"] == "akernel-test" +assert obs["AKERNEL_SNAPSHOT_OBS_USE_HTTPS"]["value"] == "true" +assert obs["AKERNEL_SNAPSHOT_OBS_PATH_STYLE"]["value"] == "false" + +for env_name, secret_key, optional in ( + ("AKERNEL_SNAPSHOT_OBS_ACCESS_KEY", "access-key", False), + ("AKERNEL_SNAPSHOT_OBS_SECRET_KEY", "secret-key", False), + ("AKERNEL_SNAPSHOT_OBS_SECURITY_TOKEN", "security-token", True), +): + assert "value" not in obs[env_name] + reference = obs[env_name]["valueFrom"]["secretKeyRef"] + assert reference["name"] == "akernel-snapshot-obs" + assert reference["key"] == secret_key + assert reference.get("optional", False) is optional + +frontend, service = frontend_documents(sys.argv[1]) +container_ports = frontend["spec"]["template"]["spec"]["containers"][0]["ports"] +assert {"name": "sandbox-router", "containerPort": 8080} in container_ports +service_ports = service["spec"]["ports"] +assert { + "name": "sandbox-router", + "port": 8080, + "targetPort": 8080, + "protocol": "TCP", +} in service_ports +PY + +echo "Kubernetes pause/resume wiring contract passed" diff --git a/deploy/akernel/values.yaml b/deploy/akernel/values.yaml index 77bbe26..eb0dc71 100644 --- a/deploy/akernel/values.yaml +++ b/deploy/akernel/values.yaml @@ -1,5 +1,18 @@ core: createNamespace: false + pauseResume: + enabled: false + snapshotStorage: + backend: datasystem + obs: + endpoint: "" + bucket: "" + existingSecret: "" + accessKeyKey: access-key + secretKeyKey: secret-key + securityTokenKey: security-token + useHttps: true + pathStyle: false monitor: enabled: true diff --git a/deploy/scripts/build-image.sh b/deploy/scripts/build-image.sh index 3993135..49e42c3 100755 --- a/deploy/scripts/build-image.sh +++ b/deploy/scripts/build-image.sh @@ -15,10 +15,35 @@ tag="" env_name="" runtime_image="" runtime_profile="${RUNTIME_PROFILE:-rrt}" -gvisor_release="" -gvisor_release_base_url="" +gvisor_release_overridden=0 +gvisor_amd64_sha512_overridden=0 +[[ -n "${GVISOR_RELEASE+x}" ]] && gvisor_release_overridden=1 +[[ -n "${GVISOR_AMD64_SHA512+x}" ]] && gvisor_amd64_sha512_overridden=1 +gvisor_release="${GVISOR_RELEASE:-release-20260706.0}" +gvisor_amd64_sha512="${GVISOR_AMD64_SHA512:-73938c145ebe554cf61a01da455688f4b732eebdf7b1b635bdef5b195868b363d8cb400e3d92ed1f377b78996805556c247a4849583910cb04e92b156053033e}" +gvisor_release_base_url="${GVISOR_RELEASE_BASE_URL:-https://storage.googleapis.com/gvisor/releases}" +otelcol_contrib_version_overridden=0 +otelcol_contrib_sha256_overridden=0 +[[ -n "${OTELCOL_CONTRIB_VERSION+x}" ]] && otelcol_contrib_version_overridden=1 +[[ -n "${OTELCOL_CONTRIB_SHA256+x}" ]] && otelcol_contrib_sha256_overridden=1 +otelcol_contrib_version="${OTELCOL_CONTRIB_VERSION:-0.120.0}" +otelcol_contrib_sha256="${OTELCOL_CONTRIB_SHA256:-81bf885bc9a86705feb3c113c5a356571390e3601eb651ffcf2b3428f6571adb}" +otelcol_contrib_url="${OTELCOL_CONTRIB_URL:-}" +open_yr_version="${OPEN_YR_VERSION:-}" open_yr_core_wheel_url="${OPEN_YR_CORE_WHEEL_URL:-}" open_yr_core_wheel_sha256="${OPEN_YR_CORE_WHEEL_SHA256:-}" +open_yr_rrt_wheel_url="${OPEN_YR_RRT_WHEEL_URL:-}" +open_yr_rrt_wheel_sha256="${OPEN_YR_RRT_WHEEL_SHA256:-}" +rrt_runtime_url="${RRT_RUNTIME_URL:-}" +rrt_runtime_sha256="${RRT_RUNTIME_SHA256:-}" +pip_index_url="${PIP_INDEX_URL:-}" +uv_python_install_mirror="${UV_PYTHON_INSTALL_MIRROR:-}" +include_kata="${AKERNEL_INCLUDE_KATA:-true}" +include_nvidia="${AKERNEL_INCLUDE_NVIDIA:-true}" +dependency_cache_dir="${AKERNEL_DEPENDENCY_CACHE_DIR:-}" +kata_release="${KATA_RELEASE:-4.0.0}" +kata_amd64_sha256="${KATA_AMD64_SHA256:-2c3b9dfeba355582b40aee462b12916c9740654d0230f696adf719d67b063a8c}" +kata_release_base_url="${KATA_RELEASE_BASE_URL:-https://github.com/kata-containers/kata-containers/releases/download}" print_component_versions=0 component_revision() { @@ -85,12 +110,36 @@ while [[ $# -gt 0 ]]; do ;; --gvisor-release) gvisor_release="$2" + gvisor_release_overridden=1 + shift 2 + ;; + --gvisor-amd64-sha512) + gvisor_amd64_sha512="$2" + gvisor_amd64_sha512_overridden=1 shift 2 ;; --gvisor-release-base-url) gvisor_release_base_url="$2" shift 2 ;; + --otelcol-contrib-version) + otelcol_contrib_version="$2" + otelcol_contrib_version_overridden=1 + shift 2 + ;; + --otelcol-contrib-sha256) + otelcol_contrib_sha256="$2" + otelcol_contrib_sha256_overridden=1 + shift 2 + ;; + --otelcol-contrib-url) + otelcol_contrib_url="$2" + shift 2 + ;; + --open-yr-version) + open_yr_version="$2" + shift 2 + ;; --open-yr-core-wheel-url) open_yr_core_wheel_url="$2" shift 2 @@ -99,6 +148,38 @@ while [[ $# -gt 0 ]]; do open_yr_core_wheel_sha256="$2" shift 2 ;; + --open-yr-rrt-wheel-url) + open_yr_rrt_wheel_url="$2" + shift 2 + ;; + --open-yr-rrt-wheel-sha256) + open_yr_rrt_wheel_sha256="$2" + shift 2 + ;; + --rrt-runtime-url) + rrt_runtime_url="$2" + shift 2 + ;; + --rrt-runtime-sha256) + rrt_runtime_sha256="$2" + shift 2 + ;; + --pip-index-url) + pip_index_url="$2" + shift 2 + ;; + --uv-python-install-mirror) + uv_python_install_mirror="$2" + shift 2 + ;; + --include-kata) + include_kata="$2" + shift 2 + ;; + --include-nvidia) + include_nvidia="$2" + shift 2 + ;; --print-component-versions) print_component_versions=1 shift @@ -113,6 +194,41 @@ case "${runtime_profile}" in rrt|python) ;; *) die "unsupported runtime profile: ${runtime_profile}; expected rrt or python" ;; esac +case "${include_kata}" in + true|false) ;; + *) die "AKERNEL_INCLUDE_KATA must be true or false" ;; +esac +case "${include_nvidia}" in + true|false) ;; + *) die "AKERNEL_INCLUDE_NVIDIA must be true or false" ;; +esac +if [[ "${gvisor_release_overridden}" != "${gvisor_amd64_sha512_overridden}" ]]; then + die "GVISOR_RELEASE and GVISOR_AMD64_SHA512 must be overridden together" +fi +if [[ "${otelcol_contrib_version_overridden}" != "${otelcol_contrib_sha256_overridden}" ]]; then + die "OTELCOL_CONTRIB_VERSION and OTELCOL_CONTRIB_SHA256 must be overridden together" +fi +[[ "${gvisor_release}" =~ ^release-[0-9A-Za-z][0-9A-Za-z.+_-]*$ ]] || \ + die "GVISOR_RELEASE must be an official release-* tag" +[[ "${gvisor_amd64_sha512}" =~ ^[0-9a-f]{128}$ ]] || \ + die "GVISOR_AMD64_SHA512 must be 128 lowercase hexadecimal characters" +[[ "${gvisor_release_base_url}" =~ ^https?://[^[:space:]]+$ ]] || \ + die "GVISOR_RELEASE_BASE_URL is invalid" +[[ "${otelcol_contrib_version}" =~ ^[0-9A-Za-z][0-9A-Za-z.+_-]*$ ]] || \ + die "OTELCOL_CONTRIB_VERSION is invalid" +[[ "${otelcol_contrib_sha256}" =~ ^[0-9a-f]{64}$ ]] || \ + die "OTELCOL_CONTRIB_SHA256 must be 64 lowercase hexadecimal characters" +if [[ -z "${otelcol_contrib_url}" ]]; then + otelcol_contrib_url="https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${otelcol_contrib_version}/otelcol-contrib_${otelcol_contrib_version}_linux_amd64.tar.gz" +fi +[[ "${otelcol_contrib_url}" =~ ^https?://[^[:space:]]+$ ]] || \ + die "OTELCOL_CONTRIB_URL is invalid" +[[ "${kata_release}" =~ ^[0-9A-Za-z][0-9A-Za-z.+_-]*$ ]] || \ + die "KATA_RELEASE is invalid" +[[ "${kata_amd64_sha256}" =~ ^[0-9a-f]{64}$ ]] || \ + die "KATA_AMD64_SHA256 must be 64 lowercase hexadecimal characters" +[[ "${kata_release_base_url}" =~ ^https?://[^[:space:]]+$ ]] || \ + die "KATA_RELEASE_BASE_URL is invalid" require_cmd docker @@ -156,9 +272,100 @@ if [[ "${print_component_versions}" == "1" ]]; then exit 0 fi +if [[ -n "${open_yr_rrt_wheel_url}" || -n "${open_yr_rrt_wheel_sha256}" ]]; then + if [[ -z "${open_yr_rrt_wheel_url}" || -z "${open_yr_rrt_wheel_sha256}" ]]; then + die "OPEN_YR_RRT_WHEEL_URL and OPEN_YR_RRT_WHEEL_SHA256 must be set together" + fi +fi +if [[ -n "${rrt_runtime_url}" || -n "${rrt_runtime_sha256}" ]]; then + if [[ -z "${rrt_runtime_url}" || -z "${rrt_runtime_sha256}" ]]; then + die "RRT_RUNTIME_URL and RRT_RUNTIME_SHA256 must be set together" + fi +fi +if [[ -n "${open_yr_rrt_wheel_url}" && -n "${rrt_runtime_url}" ]]; then + die "RRT wheel and raw runtime overrides are mutually exclusive" +fi + +if ! docker build --help | grep -q -- '--build-context'; then + die "Docker BuildKit named-context support is required" +fi + +temporary_cache_dir="" +cleanup_cache_context() { + if [[ -n "${temporary_cache_dir}" ]]; then + rm -rf -- "${temporary_cache_dir}" + fi +} +trap cleanup_cache_context EXIT + +if [[ -n "${dependency_cache_dir}" ]]; then + mkdir -p "${dependency_cache_dir}" + dependency_cache_dir="$(cd "${dependency_cache_dir}" && pwd -P)" +else + temporary_cache_dir="$(mktemp -d "${TMPDIR:-/tmp}/akernel-download-cache.XXXXXX")" + dependency_cache_dir="${temporary_cache_dir}" +fi + +if [[ "${include_kata}" == "true" && -z "${temporary_cache_dir}" ]]; then + kata_filename="kata-static-${kata_release}-amd64.tar.zst" + kata_cache_path="${dependency_cache_dir}/kata/${kata_release}/amd64/${kata_amd64_sha256}/${kata_filename}" + "${AKERNEL_REPO_ROOT}/builder/downloaders/cache-verified-download.sh" \ + "${kata_release_base_url}/${kata_release}/${kata_filename}" \ + "${kata_amd64_sha256}" \ + "${kata_cache_path}" +fi + +if [[ -z "${temporary_cache_dir}" ]]; then + gvisor_version="${gvisor_release#release-}" + gvisor_cache_path="${dependency_cache_dir}/gvisor/${gvisor_release}/x86_64/${gvisor_amd64_sha512}/runsc" + "${AKERNEL_REPO_ROOT}/builder/downloaders/cache-verified-download.sh" \ + "${gvisor_release_base_url}/release/${gvisor_version}/x86_64/runsc" \ + "${gvisor_amd64_sha512}" \ + "${gvisor_cache_path}" + + otel_filename="otelcol-contrib_${otelcol_contrib_version}_linux_amd64.tar.gz" + otel_cache_path="${dependency_cache_dir}/otelcol-contrib/${otelcol_contrib_version}/linux-amd64/${otelcol_contrib_sha256}/${otel_filename}" + "${AKERNEL_REPO_ROOT}/builder/downloaders/cache-verified-download.sh" \ + "${otelcol_contrib_url}" \ + "${otelcol_contrib_sha256}" \ + "${otel_cache_path}" +fi + +proxy_build_args=() +for proxy_name in \ + HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy; do + proxy_build_args+=(--build-arg "${proxy_name}") +done + info "building ${runtime_image} with runtime profile ${runtime_profile}" +runtime_build_args=() +if [[ -n "${open_yr_version}" ]]; then + runtime_build_args+=(--build-arg "OPEN_YR_VERSION=${open_yr_version}") +fi +if [[ -n "${pip_index_url}" ]]; then + runtime_build_args+=(--build-arg "PIP_INDEX_URL=${pip_index_url}") +fi +if [[ -n "${uv_python_install_mirror}" ]]; then + runtime_build_args+=( + --build-arg "UV_PYTHON_INSTALL_MIRROR=${uv_python_install_mirror}" + ) +fi +if [[ -n "${open_yr_rrt_wheel_url}" ]]; then + runtime_build_args+=( + --build-arg "OPEN_YR_RRT_WHEEL_URL=${open_yr_rrt_wheel_url}" + --build-arg "OPEN_YR_RRT_WHEEL_SHA256=${open_yr_rrt_wheel_sha256}" + ) +fi +if [[ -n "${rrt_runtime_url}" ]]; then + runtime_build_args+=( + --build-arg "RRT_RUNTIME_URL=${rrt_runtime_url}" + --build-arg "RRT_RUNTIME_SHA256=${rrt_runtime_sha256}" + ) +fi docker build \ -f builder/runtime.Dockerfile \ + "${proxy_build_args[@]}" \ + "${runtime_build_args[@]}" \ --target "runtime-${runtime_profile}" \ -t "${runtime_image}" \ . @@ -169,12 +376,20 @@ node_build_args=( --build-arg "AKERNEL_RUNTIME_PROFILE=${runtime_profile}" --build-arg "AKERNEL_VERSION=${akernel_version}" --build-arg "AKERNEL_REVISION=${akernel_revision}" + --build-arg "AKERNEL_INCLUDE_KATA=${include_kata}" + --build-arg "AKERNEL_INCLUDE_NVIDIA=${include_nvidia}" + --build-arg "KATA_RELEASE=${kata_release}" + --build-arg "KATA_AMD64_SHA256=${kata_amd64_sha256}" + --build-arg "KATA_RELEASE_BASE_URL=${kata_release_base_url}" + --build-arg "GVISOR_RELEASE=${gvisor_release}" + --build-arg "GVISOR_AMD64_SHA512=${gvisor_amd64_sha512}" + --build-arg "GVISOR_RELEASE_BASE_URL=${gvisor_release_base_url}" + --build-arg "OTELCOL_CONTRIB_VERSION=${otelcol_contrib_version}" + --build-arg "OTELCOL_CONTRIB_SHA256=${otelcol_contrib_sha256}" + --build-arg "OTELCOL_CONTRIB_URL=${otelcol_contrib_url}" ) -if [[ -n "${gvisor_release}" ]]; then - node_build_args+=(--build-arg "GVISOR_RELEASE=${gvisor_release}") -fi -if [[ -n "${gvisor_release_base_url}" ]]; then - node_build_args+=(--build-arg "GVISOR_RELEASE_BASE_URL=${gvisor_release_base_url}") +if [[ -n "${open_yr_version}" ]]; then + node_build_args+=(--build-arg "OPEN_YR_VERSION=${open_yr_version}") fi if [[ -n "${open_yr_core_wheel_url}" || -n "${open_yr_core_wheel_sha256}" ]]; then if [[ -z "${open_yr_core_wheel_url}" || -z "${open_yr_core_wheel_sha256}" ]]; then @@ -187,6 +402,8 @@ if [[ -n "${open_yr_core_wheel_url}" || -n "${open_yr_core_wheel_sha256}" ]]; th fi docker build \ -f builder/node.Dockerfile \ + --build-context "akernel-download-cache=${dependency_cache_dir}" \ + "${proxy_build_args[@]}" \ "${node_build_args[@]}" \ -t "${all_in_one_image}" \ . diff --git a/deploy/scripts/tests/test-build-image-rrt.sh b/deploy/scripts/tests/test-build-image-rrt.sh new file mode 100644 index 0000000..5367624 --- /dev/null +++ b/deploy/scripts/tests/test-build-image-rrt.sh @@ -0,0 +1,437 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" + +require_text() { + local file="$1" + local text="$2" + if ! grep -Fq -- "${text}" "${file}"; then + echo "missing ${text} in ${file}" >&2 + exit 1 + fi +} + +reject_text() { + local file="$1" + local text="$2" + if grep -Fq -- "${text}" "${file}"; then + echo "unexpected ${text} in ${file}" >&2 + exit 1 + fi +} + +require_text "${ROOT}/Makefile" 'OPEN_YR_RRT_WHEEL_URL ?=' +require_text "${ROOT}/Makefile" 'OPEN_YR_RRT_WHEEL_SHA256 ?=' +require_text "${ROOT}/Makefile" '--open-yr-rrt-wheel-url' +require_text "${ROOT}/Makefile" '--open-yr-rrt-wheel-sha256' +require_text "${ROOT}/Makefile" 'PIP_INDEX_URL ?=' +require_text "${ROOT}/Makefile" '--pip-index-url' +require_text "${ROOT}/Makefile" 'UV_PYTHON_INSTALL_MIRROR ?=' +require_text "${ROOT}/Makefile" '--uv-python-install-mirror' +require_text "${ROOT}/Makefile" 'AKERNEL_INCLUDE_KATA ?= true' +require_text "${ROOT}/Makefile" '--include-kata' +require_text "${ROOT}/Makefile" 'AKERNEL_INCLUDE_NVIDIA ?= true' +require_text "${ROOT}/Makefile" '--include-nvidia' +require_text "${ROOT}/Makefile" 'GVISOR_AMD64_SHA512 ?=' +require_text "${ROOT}/Makefile" '--gvisor-amd64-sha512' +require_text "${ROOT}/Makefile" 'OTELCOL_CONTRIB_VERSION ?=' +require_text "${ROOT}/Makefile" '--otelcol-contrib-version' +require_text "${ROOT}/Makefile" 'OTELCOL_CONTRIB_SHA256 ?=' +require_text "${ROOT}/Makefile" '--otelcol-contrib-sha256' +require_text "${ROOT}/Makefile" 'OTELCOL_CONTRIB_URL ?=' +require_text "${ROOT}/Makefile" '--otelcol-contrib-url' +require_text "${ROOT}/Makefile" 'version and digest must be overridden together' + +require_text "${ROOT}/deploy/scripts/build-image.sh" 'OPEN_YR_RRT_WHEEL_URL and OPEN_YR_RRT_WHEEL_SHA256 must be set together' +require_text "${ROOT}/deploy/scripts/build-image.sh" 'OPEN_YR_RRT_WHEEL_URL=${open_yr_rrt_wheel_url}' +require_text "${ROOT}/deploy/scripts/build-image.sh" 'OPEN_YR_RRT_WHEEL_SHA256=${open_yr_rrt_wheel_sha256}' +require_text "${ROOT}/deploy/scripts/build-image.sh" '--target "runtime-${runtime_profile}"' +require_text "${ROOT}/deploy/scripts/build-image.sh" 'PIP_INDEX_URL=${pip_index_url}' +require_text "${ROOT}/deploy/scripts/build-image.sh" 'UV_PYTHON_INSTALL_MIRROR=${uv_python_install_mirror}' +require_text "${ROOT}/deploy/scripts/build-image.sh" 'AKERNEL_INCLUDE_KATA=${include_kata}' +require_text "${ROOT}/deploy/scripts/build-image.sh" 'AKERNEL_INCLUDE_NVIDIA=${include_nvidia}' +require_text "${ROOT}/deploy/scripts/build-image.sh" 'AKERNEL_DEPENDENCY_CACHE_DIR' +require_text "${ROOT}/deploy/scripts/build-image.sh" 'GVISOR_RELEASE and GVISOR_AMD64_SHA512 must be overridden together' +require_text "${ROOT}/deploy/scripts/build-image.sh" 'OTELCOL_CONTRIB_VERSION and OTELCOL_CONTRIB_SHA256 must be overridden together' +require_text "${ROOT}/deploy/scripts/build-image.sh" 'release-20260706.0' +require_text "${ROOT}/deploy/scripts/build-image.sh" '73938c145ebe554cf61a01da455688f4b732eebdf7b1b635bdef5b195868b363d8cb400e3d92ed1f377b78996805556c247a4849583910cb04e92b156053033e' +require_text "${ROOT}/deploy/scripts/build-image.sh" '0.120.0' +require_text "${ROOT}/deploy/scripts/build-image.sh" '81bf885bc9a86705feb3c113c5a356571390e3601eb651ffcf2b3428f6571adb' + +require_text "${ROOT}/builder/runtime.Dockerfile" 'ARG OPEN_YR_RRT_WHEEL_URL=' +require_text "${ROOT}/builder/runtime.Dockerfile" 'COPY ./builder/downloaders/download-openyuanrong-rrt.sh /usr/local/bin/' +require_text "${ROOT}/builder/runtime.Dockerfile" '/usr/local/bin/download-openyuanrong-rrt.sh /rrt-runtime' +require_text "${ROOT}/builder/runtime.Dockerfile" 'ELF 64-bit LSB.*x86-64' +reject_text "${ROOT}/builder/runtime.Dockerfile" '-o /rrt-runtime "${RRT_RUNTIME_URL}"' +reject_text "${ROOT}/builder/runtime.Dockerfile" 'unzip -p "${wheel}" openyuanrong_rrt/rrt-runtime > /rrt-runtime' +require_text "${ROOT}/builder/downloaders/download-openyuanrong-rrt.sh" 'openyuanrong_rrt/rrt-runtime' +require_text "${ROOT}/builder/runtime.Dockerfile" '--index-url "${PIP_INDEX_URL}"' +require_text "${ROOT}/builder/runtime.Dockerfile" '--mirror "${UV_PYTHON_INSTALL_MIRROR}"' +require_text "${ROOT}/builder/runtime.Dockerfile" 'uv python install failed after 3 attempts' +require_text "${ROOT}/builder/runtime.Dockerfile" 'UV_DEFAULT_INDEX="${PIP_INDEX_URL}" uv venv' +if [[ "$(grep -Fc 'ARG PIP_INDEX_URL' "${ROOT}/builder/runtime.Dockerfile")" -ne 1 ]]; then + echo "builder/runtime.Dockerfile must declare PIP_INDEX_URL once in python-runtime-rootfs" >&2 + exit 1 +fi + +require_text "${ROOT}/builder/node.Dockerfile" 'ARG AKERNEL_RUNTIME_PROFILE=rrt' +require_text "${ROOT}/builder/node.Dockerfile" '.akernel-rrt-capable' +reject_text "${ROOT}/builder/node.Dockerfile" 'AKERNEL_ENABLE_RRT_RUNTIME' +reject_text "${ROOT}/builder/node.Dockerfile" 'rrt) services=/tmp/yr_services_rrt.yaml; touch' +require_text "${ROOT}/builder/node.Dockerfile" 'if [ -n "${OPEN_YR_CORE_WHEEL_SHA256}" ]; then' +require_text "${ROOT}/builder/node.Dockerfile" 'COPY ./builder/downloaders/download-openyuanrong-core.sh /usr/local/bin/' +require_text "${ROOT}/builder/node.Dockerfile" '/usr/local/bin/download-openyuanrong-core.sh "${download_dir}"' +require_text "${ROOT}/builder/node.Dockerfile" 'set -- "${download_dir}"/*.whl' +require_text "${ROOT}/builder/node.Dockerfile" 'test "$#" -eq 1' +reject_text "${ROOT}/builder/node.Dockerfile" 'wheel_url="${OPEN_YR_RELEASE_BASE_URL}' +require_text "${ROOT}/builder/node.Dockerfile" 'yr_services_python.yaml' +require_text "${ROOT}/builder/node.Dockerfile" 'ARG AKERNEL_INCLUDE_KATA=true' +require_text "${ROOT}/builder/node.Dockerfile" 'if [ "${AKERNEL_INCLUDE_KATA}" = "false" ]; then' +require_text "${ROOT}/builder/node.Dockerfile" 'from=akernel-download-cache' +require_text "${ROOT}/builder/node.Dockerfile" 'target=/var/cache/akernel-downloads,ro' +require_text "${ROOT}/builder/node.Dockerfile" 'kata-cache-hit' +require_text "${ROOT}/builder/node.Dockerfile" 'gvisor-cache-hit' +require_text "${ROOT}/builder/node.Dockerfile" 'otelcol-cache-hit' +require_text "${ROOT}/builder/node.Dockerfile" 'echo "${GVISOR_AMD64_SHA512} runsc" | sha512sum -c -' +require_text "${ROOT}/builder/node.Dockerfile" 'echo "${OTELCOL_CONTRIB_SHA256} ${archive}" | sha256sum -c -' +if [[ "$(grep -Fc 'from=akernel-download-cache,target=/var/cache/akernel-downloads,ro' "${ROOT}/builder/node.Dockerfile")" -ne 3 ]]; then + echo "all three dependency consumers must mount akernel-download-cache read-only" >&2 + exit 1 +fi +require_text "${ROOT}/builder/node.Dockerfile" 'ARG AKERNEL_INCLUDE_NVIDIA=true' +require_text "${ROOT}/builder/node.Dockerfile" 'if [ "${AKERNEL_INCLUDE_NVIDIA}" = "false" ]; then' +require_text "${ROOT}/builder/node.Dockerfile" ' patch \' +require_text "${ROOT}/builder/node.Dockerfile" 'openyuanrong-core-454473b64447-pause-resume-process.patch' + +[[ -x "${ROOT}/builder/downloaders/download-openyuanrong-core.sh" ]] || { + echo "core downloader must be executable" >&2 + exit 1 +} +[[ -x "${ROOT}/builder/downloaders/download-openyuanrong-rrt.sh" ]] || { + echo "RRT downloader must be executable" >&2 + exit 1 +} + +require_text "${ROOT}/builder/config/yr_services.yaml" 'rrt:' +require_text "${ROOT}/builder/config/yr_services.yaml" 'runtime: rust' +require_text "${ROOT}/builder/config/yr_services.yaml" '/__yuanrong/usr/local/bin/rrt-runtime' + +behavior_tmp="$(mktemp -d)" +trap 'rm -rf "${behavior_tmp}"' EXIT +fixture="${behavior_tmp}/fixture" +mkdir -p \ + "${fixture}/deploy/scripts" \ + "${fixture}/builder/downloaders" \ + "${fixture}/src/sandboxd/version" \ + "${fixture}/src/distill-fs/src" \ + "${behavior_tmp}/bin" +cp "${ROOT}/deploy/scripts/build-image.sh" "${fixture}/deploy/scripts/" +cp "${ROOT}/deploy/scripts/common.sh" "${fixture}/deploy/scripts/" +cp "${ROOT}/builder/downloaders/cache-verified-download.sh" \ + "${fixture}/builder/downloaders/" +: >"${fixture}/builder/runtime.Dockerfile" +: >"${fixture}/builder/node.Dockerfile" +printf '1.2.3\n' >"${fixture}/src/sandboxd/version/VERSION" +printf '[package]\nname = "distill_fs"\nversion = "4.5.6"\n' \ + >"${fixture}/src/distill-fs/Cargo.toml" +printf 'fn main() {}\n' >"${fixture}/src/distill-fs/src/main.rs" + +for repository in \ + "${fixture}" \ + "${fixture}/src/sandboxd" \ + "${fixture}/src/distill-fs"; do + git -C "${repository}" init -q + git -C "${repository}" config user.email build-test@example.invalid + git -C "${repository}" config user.name 'Build Test' + git -C "${repository}" add . + git -C "${repository}" commit -qm fixture +done + +cat >"${behavior_tmp}/bin/docker" <<'EOF' +#!/usr/bin/env bash +if [[ "$*" == "build --help" ]]; then + echo " --build-context stringArray" + exit 0 +fi +printf '%s\n' "$*" >>"${DOCKER_LOG}" +EOF +chmod +x "${behavior_tmp}/bin/docker" + +cat >"${behavior_tmp}/bin/curl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +destination="" +url="" +while [[ "$#" -gt 0 ]]; do + case "$1" in + -o) + destination="$2" + shift 2 + ;; + http://*|https://*) + url="$1" + shift + ;; + *) + shift + ;; + esac +done +[[ -n "${destination}" ]] +[[ -n "${url}" ]] +mkdir -p "${FAKE_CURL_CALL_DIR}" +: >"${FAKE_CURL_CALL_DIR}/call-$$-${RANDOM}" +printf '%s\n' "${url}" >>"${FAKE_CURL_URL_LOG}" +cp "${FAKE_CURL_SOURCE}" "${destination}" +EOF +chmod +x "${behavior_tmp}/bin/curl" + +runtime_sha='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +dependency_source="${behavior_tmp}/dependency-source" +printf 'one shared dependency fixture\n' >"${dependency_source}" +kata_sha="$(sha256sum "${dependency_source}" | awk '{print $1}')" +gvisor_sha512="$(sha512sum "${dependency_source}" | awk '{print $1}')" +otel_sha256="$(sha256sum "${dependency_source}" | awk '{print $1}')" +dependency_cache="${behavior_tmp}/dependency-cache" +mkdir -p "${dependency_cache}" +dependency_cache="$(cd "${dependency_cache}" && pwd -P)" +fake_curl_calls="${behavior_tmp}/fake-curl-calls" +fake_curl_urls="${behavior_tmp}/fake-curl-urls" +build_output="$( + DOCKER_LOG="${behavior_tmp}/docker.log" \ + PATH="${behavior_tmp}/bin:${ROOT}/builder/downloaders/tests/fixtures:${PATH}" \ + AKERNEL_DEPENDENCY_CACHE_DIR="${dependency_cache}" \ + KATA_RELEASE=9.9.9 \ + KATA_AMD64_SHA256="${kata_sha}" \ + KATA_RELEASE_BASE_URL=https://example.invalid/kata/releases/download \ + FAKE_CURL_SOURCE="${dependency_source}" \ + FAKE_CURL_CALL_DIR="${fake_curl_calls}" \ + FAKE_CURL_URL_LOG="${fake_curl_urls}" \ + "${fixture}/deploy/scripts/build-image.sh" \ + --repository registry.example.invalid/akernel \ + --tag behavior-test \ + --runtime-profile rrt \ + --gvisor-release release-9.9.9 \ + --gvisor-amd64-sha512 "${gvisor_sha512}" \ + --gvisor-release-base-url https://example.invalid/gvisor/releases \ + --otelcol-contrib-version 8.8.8 \ + --otelcol-contrib-sha256 "${otel_sha256}" \ + --open-yr-version 0.8.1 \ + --rrt-runtime-url https://artifacts.example.invalid/rrt-runtime-amd64 \ + --rrt-runtime-sha256 "${runtime_sha}" +)" + +kata_cache_file="${dependency_cache}/kata/9.9.9/amd64/${kata_sha}/kata-static-9.9.9-amd64.tar.zst" +gvisor_cache_file="${dependency_cache}/gvisor/release-9.9.9/x86_64/${gvisor_sha512}/runsc" +otel_cache_file="${dependency_cache}/otelcol-contrib/8.8.8/linux-amd64/${otel_sha256}/otelcol-contrib_8.8.8_linux_amd64.tar.gz" +for cache_file in "${kata_cache_file}" "${gvisor_cache_file}" "${otel_cache_file}"; do + [[ "${build_output}" == *"cache-fill ${cache_file}"* ]] || { + echo "first cached build did not report a cache fill for ${cache_file}" >&2 + exit 1 + } + cmp "${dependency_source}" "${cache_file}" +done +[[ "$(find "${fake_curl_calls}" -type f -name 'call-*' | wc -l | tr -d ' ')" == "3" ]] || { + echo "first cached build did not download exactly three dependencies" >&2 + exit 1 +} +[[ "$(sed -n '1p' "${fake_curl_urls}")" == \ + 'https://example.invalid/kata/releases/download/9.9.9/kata-static-9.9.9-amd64.tar.zst' ]] || { + echo "Kata host prefetch used the wrong URL" >&2 + exit 1 +} +[[ "$(sed -n '2p' "${fake_curl_urls}")" == \ + 'https://example.invalid/gvisor/releases/release/9.9.9/x86_64/runsc' ]] || { + echo "gVisor host prefetch did not strip the release- prefix" >&2 + exit 1 +} +[[ "$(sed -n '3p' "${fake_curl_urls}")" == \ + 'https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v8.8.8/otelcol-contrib_8.8.8_linux_amd64.tar.gz' ]] || { + echo "OpenTelemetry host prefetch did not derive the versioned default URL" >&2 + exit 1 +} + +runtime_invocation="$(sed -n '1p' "${behavior_tmp}/docker.log")" +node_invocation="$(sed -n '2p' "${behavior_tmp}/docker.log")" +for invocation in "${runtime_invocation}" "${node_invocation}"; do + for proxy_name in \ + HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy; do + [[ "${invocation}" == *"--build-arg ${proxy_name}"* ]] || { + echo "Docker invocation is missing proxy build arg ${proxy_name}" >&2 + exit 1 + } + done +done +for expected in \ + 'OPEN_YR_VERSION=0.8.1' \ + 'RRT_RUNTIME_URL=https://artifacts.example.invalid/rrt-runtime-amd64' \ + "RRT_RUNTIME_SHA256=${runtime_sha}"; do + [[ "${runtime_invocation}" == *"${expected}"* ]] || { + echo "runtime Docker invocation is missing ${expected}" >&2 + exit 1 + } +done +[[ "${node_invocation}" == *'OPEN_YR_VERSION=0.8.1'* ]] || { + echo "node Docker invocation is missing OPEN_YR_VERSION" >&2 + exit 1 +} +[[ "${node_invocation}" != *'RRT_RUNTIME_URL='* ]] || { + echo "node Docker invocation unexpectedly contains RRT_RUNTIME_URL" >&2 + exit 1 +} + +for expected in \ + "--build-context akernel-download-cache=${dependency_cache}" \ + 'KATA_RELEASE=9.9.9' \ + "KATA_AMD64_SHA256=${kata_sha}" \ + 'KATA_RELEASE_BASE_URL=https://example.invalid/kata/releases/download' \ + 'GVISOR_RELEASE=release-9.9.9' \ + "GVISOR_AMD64_SHA512=${gvisor_sha512}" \ + 'GVISOR_RELEASE_BASE_URL=https://example.invalid/gvisor/releases' \ + 'OTELCOL_CONTRIB_VERSION=8.8.8' \ + "OTELCOL_CONTRIB_SHA256=${otel_sha256}" \ + 'OTELCOL_CONTRIB_URL=https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v8.8.8/otelcol-contrib_8.8.8_linux_amd64.tar.gz'; do + [[ "${node_invocation}" == *"${expected}"* ]] || { + echo "node Docker invocation is missing ${expected}" >&2 + exit 1 + } +done + +second_output="$( + DOCKER_LOG="${behavior_tmp}/docker.log" \ + PATH="${behavior_tmp}/bin:${ROOT}/builder/downloaders/tests/fixtures:${PATH}" \ + AKERNEL_DEPENDENCY_CACHE_DIR="${dependency_cache}" \ + KATA_RELEASE=9.9.9 \ + KATA_AMD64_SHA256="${kata_sha}" \ + KATA_RELEASE_BASE_URL=https://example.invalid/kata/releases/download \ + FAKE_CURL_SOURCE="${dependency_source}" \ + FAKE_CURL_CALL_DIR="${fake_curl_calls}" \ + FAKE_CURL_URL_LOG="${fake_curl_urls}" \ + "${fixture}/deploy/scripts/build-image.sh" \ + --repository registry.example.invalid/akernel \ + --tag behavior-test-hit \ + --runtime-profile rrt \ + --gvisor-release release-9.9.9 \ + --gvisor-amd64-sha512 "${gvisor_sha512}" \ + --gvisor-release-base-url https://example.invalid/gvisor/releases \ + --otelcol-contrib-version 8.8.8 \ + --otelcol-contrib-sha256 "${otel_sha256}" \ + --open-yr-version 0.8.1 \ + --rrt-runtime-url https://artifacts.example.invalid/rrt-runtime-amd64 \ + --rrt-runtime-sha256 "${runtime_sha}" +)" +for cache_file in "${kata_cache_file}" "${gvisor_cache_file}" "${otel_cache_file}"; do + [[ "${second_output}" == *"cache-hit ${cache_file}"* ]] || { + echo "second cached build did not report a cache hit for ${cache_file}" >&2 + exit 1 + } +done +[[ "$(find "${fake_curl_calls}" -type f -name 'call-*' | wc -l | tr -d ' ')" == "3" ]] || { + echo "cache hits unexpectedly downloaded dependencies" >&2 + exit 1 +} +[[ "$(wc -l <"${fake_curl_urls}" | tr -d ' ')" == "3" ]] || { + echo "cache hits unexpectedly invoked host curl" >&2 + exit 1 +} + +uncached_log="${behavior_tmp}/uncached-docker.log" +DOCKER_LOG="${uncached_log}" \ +PATH="${behavior_tmp}/bin:${ROOT}/builder/downloaders/tests/fixtures:${PATH}" \ +FAKE_CURL_FAIL=1 \ +FAKE_CURL_SOURCE="${dependency_source}" \ +FAKE_CURL_CALL_DIR="${behavior_tmp}/uncached-calls" \ + "${fixture}/deploy/scripts/build-image.sh" \ + --repository registry.example.invalid/akernel \ + --tag behavior-test-uncached \ + --runtime-profile rrt \ + --gvisor-release-base-url https://mirror.example.invalid/gvisor \ + --otelcol-contrib-url https://mirror.example.invalid/otelcol.tar.gz \ + --open-yr-version 0.8.1 +uncached_node_invocation="$(sed -n '2p' "${uncached_log}")" +[[ "${uncached_node_invocation}" == *'--build-context akernel-download-cache='* ]] || { + echo "uncached node build is missing the empty named context" >&2 + exit 1 +} +[[ ! -d "${behavior_tmp}/uncached-calls" ]] || { + echo "uncached build unexpectedly invoked the host downloader" >&2 + exit 1 +} + +for expected in \ + 'GVISOR_RELEASE=release-20260706.0' \ + 'GVISOR_AMD64_SHA512=73938c145ebe554cf61a01da455688f4b732eebdf7b1b635bdef5b195868b363d8cb400e3d92ed1f377b78996805556c247a4849583910cb04e92b156053033e' \ + 'GVISOR_RELEASE_BASE_URL=https://mirror.example.invalid/gvisor' \ + 'OTELCOL_CONTRIB_VERSION=0.120.0' \ + 'OTELCOL_CONTRIB_SHA256=81bf885bc9a86705feb3c113c5a356571390e3601eb651ffcf2b3428f6571adb' \ + 'OTELCOL_CONTRIB_URL=https://mirror.example.invalid/otelcol.tar.gz'; do + [[ "${uncached_node_invocation}" == *"${expected}"* ]] || { + echo "uncached node Docker invocation is missing ${expected}" >&2 + exit 1 + } +done + +for incomplete_override in \ + '--gvisor-release release-9.9.9' \ + '--gvisor-amd64-sha512 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ + '--otelcol-contrib-version 8.8.8' \ + '--otelcol-contrib-sha256 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; do + override_log="${behavior_tmp}/override-$(printf '%s' "${incomplete_override}" | tr -cs '[:alnum:]' '-').log" + if DOCKER_LOG="${override_log}" PATH="${behavior_tmp}/bin:${PATH}" \ + "${fixture}/deploy/scripts/build-image.sh" \ + --repository registry.example.invalid/akernel \ + --tag invalid-override \ + ${incomplete_override} >/dev/null 2>&1; then + echo "incomplete override unexpectedly succeeded: ${incomplete_override}" >&2 + exit 1 + fi + [[ ! -e "${override_log}" ]] || { + echo "incomplete override reached Docker: ${incomplete_override}" >&2 + exit 1 + } +done + +expect_rejected_before_docker() { + local label="$1" + shift + local validation_log="${behavior_tmp}/validation-${label}.log" + if DOCKER_LOG="${validation_log}" PATH="${behavior_tmp}/bin:${PATH}" \ + "${fixture}/deploy/scripts/build-image.sh" \ + --repository registry.example.invalid/akernel \ + --tag "invalid-${label}" \ + "$@" >/dev/null 2>&1; then + echo "invalid ${label} unexpectedly succeeded" >&2 + exit 1 + fi + [[ ! -e "${validation_log}" ]] || { + echo "invalid ${label} reached Docker" >&2 + exit 1 + } +} + +expect_rejected_before_docker gvisor-release \ + --gvisor-release 9.9.9 \ + --gvisor-amd64-sha512 "${gvisor_sha512}" +expect_rejected_before_docker gvisor-sha512 \ + --gvisor-release release-9.9.9 \ + --gvisor-amd64-sha512 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +expect_rejected_before_docker gvisor-sha512-length \ + --gvisor-release release-9.9.9 \ + --gvisor-amd64-sha512 "${gvisor_sha512%?}" +expect_rejected_before_docker gvisor-url \ + --gvisor-release-base-url ftp://example.invalid/gvisor +expect_rejected_before_docker otel-version \ + --otelcol-contrib-version 8/8/8 \ + --otelcol-contrib-sha256 "${otel_sha256}" +expect_rejected_before_docker otel-sha256 \ + --otelcol-contrib-version 8.8.8 \ + --otelcol-contrib-sha256 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +expect_rejected_before_docker otel-sha256-length \ + --otelcol-contrib-version 8.8.8 \ + --otelcol-contrib-sha256 "${otel_sha256%?}" +expect_rejected_before_docker otel-url \ + --otelcol-contrib-url file:///tmp/otelcol.tar.gz + +echo "RRT build contract checks passed" diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index ecd210f..06cfca5 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -203,6 +203,24 @@ variable to test another registry, tag, or locally built image: IMAGE=":" ./start.sh ``` +Images built with `OPEN_YR_RRT_WHEEL_URL` and +`OPEN_YR_RRT_WHEEL_SHA256` contain the Rust runtime required by sandbox +pause/resume. Enable the standalone control-plane wiring explicitly: + +```bash +AKERNEL_ENABLE_PAUSE_RESUME=true IMAGE="" ./start.sh +``` + +Startup fails closed when the flag is enabled on an image without the RRT +capability marker. Checkpoints are stored below +`data/sandboxd/root/checkpoints`. Run the acceptance test with the exact +pause/resume SDK wheel after exporting `YR_SERVER_ADDRESS`, `YR_TOKEN`, and +`YR_TLS=1`: + +```bash +python3 pause_resume_e2e.py --report data/pause-resume-e2e.json +``` + The gateway defaults to `traefik:v3.6.8`. Override it independently when needed: diff --git a/deploy/standalone/pause_resume_e2e.py b/deploy/standalone/pause_resume_e2e.py new file mode 100644 index 0000000..76dbbf1 --- /dev/null +++ b/deploy/standalone/pause_resume_e2e.py @@ -0,0 +1,711 @@ +#!/usr/bin/env python3 +"""Run the standalone RRT pause/resume acceptance test and write JSON evidence.""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import subprocess +import sys +import time +import traceback +import urllib.error +import urllib.request +from dataclasses import asdict +from pathlib import Path +from typing import Any, Callable + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--report", type=Path, required=True) + parser.add_argument("--evidence-dir", type=Path, required=True) + parser.add_argument("--node-container", default="akernel-node") + parser.add_argument("--memory", type=int, default=4096) + parser.add_argument("--create-timeout", type=int, default=240) + parser.add_argument("--proxy-restart-timeout", type=int, default=180) + args = parser.parse_args() + + for name in ("YR_SERVER_ADDRESS", "YR_GATEWAY_ADDRESS", "YR_TOKEN"): + if not os.environ.get(name, "").strip(): + parser.error(f"{name} is required") + + from yr_sandbox import Sandbox + + started = time.time() + report: dict[str, Any] = { + "schemaVersion": 1, + "startedAtUnix": int(started), + "result": "failed", + "steps": [], + } + args.evidence_dir.mkdir(parents=True, exist_ok=True) + report["evidenceDirectory"] = str(args.evidence_dir) + report["nodeContainer"] = args.node_container + sandbox: Any = None + deleted = False + public_url = "" + watch_process, watch_stream = _start_etcd_watch( + args.node_container, args.evidence_dir / "etcd-watch.jsonl" + ) + + def step(name: str, operation: Callable[[], Any]) -> Any: + step_started = time.monotonic() + try: + value = operation() + except Exception as exc: + report["steps"].append( + { + "name": name, + "result": "failed", + "durationSeconds": round(time.monotonic() - step_started, 3), + "errorType": type(exc).__name__, + "error": str(exc), + } + ) + raise + report["steps"].append( + { + "name": name, + "result": "passed", + "durationSeconds": round(time.monotonic() - step_started, 3), + } + ) + print(f"[PASS] {name}", flush=True) + return value + + marker_path = "/tmp/akernel-pause-resume-marker.txt" + marker_value = "akernel-rrt-pause-resume-v1" + public_port = 18080 + public_body = "akernel-sandboxrouter-resume-ok" + public_ready = "/tmp/akernel-public-http-ready" + public_server = rf'''perl -MSocket -e '$|=1; socket(S,PF_INET,SOCK_STREAM,getprotobyname("tcp")); setsockopt(S,SOL_SOCKET,SO_REUSEADDR,1); bind(S,sockaddr_in({public_port},INADDR_ANY)) or die $!; listen(S,10); open(F, ">", "{public_ready}") or die $!; print F "ready"; close F; while(accept(C,S)){{ print C "HTTP/1.1 200 OK\r\nContent-Length: {len(public_body)}\r\nConnection: close\r\n\r\n{public_body}"; close C; }}' ''' + try: + sandbox = step( + "create RRT sandbox", + lambda: Sandbox( + name=f"akernel-pause-resume-{int(started)}", + runtime="runsc", + cpu=1000, + memory=args.memory, + create_timeout=args.create_timeout, + schedule_timeout=180, + port_forwardings=[public_port], + ), + ) + report["sandboxId"] = sandbox.id + report["runtime"] = "RRT/Rust Runtime" + report["sandboxBackend"] = "runsc" + report["gatewayAddress"] = os.environ["YR_GATEWAY_ADDRESS"] + + step("write marker through RRT", lambda: sandbox.files.write(marker_path, marker_value)) + step( + "read marker before pause", + lambda: _assert_equal(sandbox.files.read(marker_path), marker_value), + ) + step( + "execute command before pause", + lambda: _assert_command(sandbox.commands.run("printf pre-pause-ok"), "pre-pause-ok"), + ) + handle = step( + "start stdin-blocked process", + lambda: sandbox.commands.run( + "read value; printf 'resumed:%s' \"$value\"", + background=True, + stdin=True, + ), + ) + step( + "start public HTTP server", + lambda: sandbox.commands.run(public_server, background=True), + ) + step( + "observe public HTTP process readiness", + lambda: _assert_command( + sandbox.commands.run( + f"while [ ! -f {public_ready} ]; do sleep 0.05; done; printf ready" + ), + "ready", + ), + ) + public_url = sandbox.get_port_url(public_port) + report["publicPort"] = public_port + report["publicURL"] = public_url + step( + "public HTTP before pause uses SandboxRouter", + lambda: _assert_equal(_fetch_public_once(public_url), public_body), + ) + source_authority = step( + "capture RUNNING source authority", + lambda: _capture_authority( + args.node_container, sandbox.id, "source-running", args.evidence_dir + ), + ) + step( + "verify source RUNNING ETCD authority", + lambda: _assert_running_authority(source_authority), + ) + source_instance = source_authority["instance"]["value"] + source_runtime_id = source_instance["runtimeID"] + source_container_id = source_instance["containerID"] + source_mappings = _port_mappings_from_instance(source_instance) + report["source"] = { + "runtimeId": source_runtime_id, + "containerId": source_container_id, + "functionProxyId": source_instance["functionProxyID"], + "portMappings": source_mappings, + } + source_inspect = step( + "capture source sandboxd physical fact", + lambda: _inspect_sandboxd( + args.node_container, + source_container_id, + args.evidence_dir / "sandboxd-source-running.json", + ), + ) + step( + "verify source mappings match sandboxd physical fact", + lambda: _assert_equal( + _port_mappings_from_sandboxd(source_inspect), source_mappings + ), + ) + source_proxy_pid = _function_proxy_pid(args.node_container) + source_registration_count = _function_proxy_registration_count(args.node_container) + report["source"]["functionProxyPid"] = source_proxy_pid + report["source"]["registrationCount"] = source_registration_count + + pause_result = step("pause sandbox", lambda: sandbox.pause(ttl_seconds=1800)) + report["pause"] = asdict(pause_result) + report["snapshotId"] = pause_result.snapshot_id + step( + "observe paused state", + lambda: _assert_equal(sandbox.get_info().state, "paused"), + ) + paused_authority = step( + "capture PAUSED authority", + lambda: _capture_authority( + args.node_container, sandbox.id, "paused", args.evidence_dir + ), + ) + step( + "verify PAUSED ETCD authority", + lambda: _assert_paused_authority( + paused_authority, pause_result.snapshot_id, source_instance + ), + ) + released_source = step( + "verify source runtime physical fact released", + lambda: _inspect_sandboxd( + args.node_container, + source_container_id, + args.evidence_dir / "sandboxd-source-after-pause.json", + ), + ) + _assert_equal(released_source, None) + step( + "SandboxRouter rejects public data while paused", + lambda: _assert_public_paused_once(public_url), + ) + target_proxy_pid = step( + "terminate source FunctionProxy after PAUSED commit", + lambda: _restart_function_proxy( + args.node_container, + source_proxy_pid, + source_registration_count, + args.proxy_restart_timeout, + ), + ) + report["sourceProxyExit"] = { + "sourcePid": source_proxy_pid, + "targetPid": target_proxy_pid, + "proven": target_proxy_pid != source_proxy_pid, + } + + resume_result = step("resume sandbox", sandbox.resume) + report["resume"] = asdict(resume_result) + if str(public_port) not in resume_result.port_mappings: + raise AssertionError( + "resume result is missing the public port mapping for " + f"container port {public_port}: {resume_result.port_mappings!r}" + ) + step( + "observe running state", + lambda: _assert_equal(sandbox.get_info().state, "running"), + ) + step( + "first file request after resume", + lambda: _assert_equal(sandbox.files.read(marker_path), marker_value), + ) + step( + "first exec request after resume", + lambda: _assert_command(sandbox.commands.run("printf post-resume-ok"), "post-resume-ok"), + ) + step( + "first public request after resume uses SandboxRouter", + lambda: _assert_equal(_fetch_public_once(public_url), public_body), + ) + winner_authority = step( + "capture RUNNING winner authority", + lambda: _capture_authority( + args.node_container, sandbox.id, "winner-running", args.evidence_dir + ), + ) + step( + "verify RUNNING winner ETCD authority", + lambda: _assert_running_authority(winner_authority), + ) + winner_instance = winner_authority["instance"]["value"] + winner_mappings = _port_mappings_from_instance(winner_instance) + step( + "verify resume mappings match ETCD winner", + lambda: _assert_equal( + _normalize_port_mappings(resume_result.port_mappings), winner_mappings + ), + ) + winner_runtime_id = winner_instance["runtimeID"] + winner_container_id = winner_instance["containerID"] + winner_inspect = step( + "verify sandboxd physical fact after resume", + lambda: _inspect_sandboxd( + args.node_container, + winner_container_id, + args.evidence_dir / "sandboxd-winner-running.json", + ), + ) + if winner_inspect is None: + raise AssertionError("sandboxd has no exact winner sandbox after resume") + _assert_equal(_port_mappings_from_sandboxd(winner_inspect), winner_mappings) + report["winner"] = { + "runtimeId": winner_runtime_id, + "containerId": winner_container_id, + "functionProxyId": winner_instance["functionProxyID"], + "functionProxyPid": target_proxy_pid, + "portMappings": winner_mappings, + "sourcePortsMayBeReused": True, + } + step( + "verify exact snapshot cleanup", + lambda: _assert_snapshot_cleaned(winner_authority, pause_result.snapshot_id), + ) + step("send stdin after resume", lambda: handle.send_stdin("continuity\n", eof=True)) + step( + "verify process continuity", + lambda: _assert_command(handle.wait(timeout=90), "resumed:continuity"), + ) + step( + "verify direct route remains active", + lambda: _assert_equal(sandbox._client._direct_disabled, False), + ) + step("delete sandbox", sandbox.kill) + deleted = True + cleanup_authority = step( + "verify ETCD and sandboxd cleanup", + lambda: _verify_cleanup( + args.node_container, + sandbox.id, + winner_container_id, + args.evidence_dir, + ), + ) + report["cleanupAuthority"] = cleanup_authority + step( + "SandboxRouter returns missing after delete", + lambda: _assert_public_missing_once(public_url), + ) + report["cleanup"] = "passed" + report["result"] = "passed" + except Exception as exc: + report["errorType"] = type(exc).__name__ + report["error"] = str(exc) + report["traceback"] = traceback.format_exc() + finally: + if sandbox is not None and not deleted: + try: + sandbox.kill() + report["cleanup"] = "passed" + except Exception as exc: + report["cleanup"] = "failed" + report["cleanupError"] = f"{type(exc).__name__}: {exc}" + report["result"] = "failed" + _stop_etcd_watch(watch_process, watch_stream) + report["finishedAtUnix"] = int(time.time()) + report["durationSeconds"] = round(time.time() - started, 3) + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + + if report["result"] != "passed": + print(json.dumps({"result": report["result"], "error": report.get("error")}), file=sys.stderr) + return 1 + print(json.dumps({"result": "passed", "report": str(args.report)})) + return 0 + + +def _assert_equal(actual: Any, expected: Any) -> None: + if actual != expected: + raise AssertionError(f"expected {expected!r}, got {actual!r}") + + +def _assert_command(result: Any, expected_stdout: str) -> None: + if result.exit_code != 0 or result.stdout != expected_stdout: + raise AssertionError( + f"expected rc=0 stdout={expected_stdout!r}, " + f"got rc={result.exit_code} stdout={result.stdout!r} stderr={result.stderr!r}" + ) + + +def _run(command: list[str], *, check: bool = True) -> subprocess.CompletedProcess[str]: + completed = subprocess.run(command, text=True, capture_output=True) + if check and completed.returncode != 0: + raise RuntimeError( + f"command failed ({completed.returncode}): {' '.join(command)}\n" + f"stdout: {completed.stdout}\nstderr: {completed.stderr}" + ) + return completed + + +def _docker_exec(node_container: str, *command: str, check: bool = True) -> subprocess.CompletedProcess[str]: + return _run(["docker", "exec", node_container, *command], check=check) + + +def _node_ip(node_container: str) -> str: + result = _run( + [ + "docker", + "inspect", + "--format", + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", + node_container, + ] + ) + address = result.stdout.strip() + if not address: + raise RuntimeError(f"container {node_container} has no bridge address") + return address + + +def _etcdctl_command(node_container: str, *arguments: str) -> list[str]: + endpoint = f"http://{_node_ip(node_container)}:2379" + return [ + "docker", + "exec", + "-e", + "ETCDCTL_API=3", + node_container, + "/home/yuanrong/third_party/etcd/etcdctl", + f"--endpoints={endpoint}", + *arguments, + ] + + +def _start_etcd_watch( + node_container: str, evidence_path: Path +) -> tuple[subprocess.Popen[bytes], Any]: + stream = evidence_path.open("wb") + process = subprocess.Popen( + _etcdctl_command(node_container, "watch", "/", "--prefix", "--write-out=json"), + stdout=stream, + stderr=subprocess.STDOUT, + ) + return process, stream + + +def _stop_etcd_watch(process: subprocess.Popen[bytes], stream: Any) -> None: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + stream.close() + + +def _decode_etcd_records(raw: str) -> list[dict[str, Any]]: + payload = json.loads(raw) + records: list[dict[str, Any]] = [] + for item in payload.get("kvs", []): + key = base64.b64decode(item["key"]).decode(errors="replace") + value_text = base64.b64decode(item["value"]).decode(errors="replace") + try: + value: Any = json.loads(value_text) + except json.JSONDecodeError: + value = value_text + records.append( + { + "key": key, + "value": value, + "createRevision": item.get("create_revision"), + "modRevision": item.get("mod_revision"), + "version": item.get("version"), + } + ) + return records + + +def _capture_authority( + node_container: str, sandbox_id: str, phase: str, evidence_dir: Path +) -> dict[str, Any]: + completed = _run( + _etcdctl_command(node_container, "get", "/", "--prefix", "--write-out=json") + ) + records = _decode_etcd_records(completed.stdout) + matching = [ + record + for record in records + if isinstance(record["value"], dict) + and record["value"].get("instanceID") == sandbox_id + ] + instance = next( + (record for record in matching if "/sn/instance/" in record["key"]), None + ) + route = next((record for record in matching if "/yr/route/" in record["key"]), None) + authority = { + "phase": phase, + "sandboxId": sandbox_id, + "instance": instance, + "route": route, + "matchingRecords": matching, + } + (evidence_dir / f"etcd-{phase}.json").write_text( + json.dumps(authority, indent=2, sort_keys=True) + "\n" + ) + return authority + + +def _state_code(record: dict[str, Any]) -> int: + return int(record.get("instanceStatus", {}).get("code", 0)) + + +def _assert_running_authority(authority: dict[str, Any]) -> None: + if authority["instance"] is None or authority["route"] is None: + raise AssertionError(f"RUNNING authority is incomplete: {authority!r}") + instance = authority["instance"]["value"] + route = authority["route"]["value"] + if _state_code(instance) != 3 or _state_code(route) != 3: + raise AssertionError(f"expected RUNNING(3), got instance={instance!r} route={route!r}") + owner = instance.get("functionProxyID", "") + if not owner or owner == "InstanceManagerOwner" or route.get("functionProxyID") != owner: + raise AssertionError(f"RUNNING owner/route mismatch: instance={instance!r} route={route!r}") + for field in ("runtimeID", "functionAgentID", "containerID", "proxyGrpcAddress"): + if not instance.get(field): + raise AssertionError(f"RUNNING winner is missing {field}: {instance!r}") + if route.get("proxyGrpcAddress") != instance.get("proxyGrpcAddress"): + raise AssertionError("RUNNING route does not publish the winner proxyGrpcAddress") + if not _port_mappings_from_instance(instance): + raise AssertionError("RUNNING authority has no target portForward mapping") + + +def _assert_paused_authority( + authority: dict[str, Any], snapshot_id: str, source: dict[str, Any] +) -> None: + if authority["instance"] is None or authority["route"] is None: + raise AssertionError(f"PAUSED authority is incomplete: {authority!r}") + instance = authority["instance"]["value"] + route = authority["route"]["value"] + if _state_code(instance) != 13 or _state_code(route) != 13: + raise AssertionError(f"expected PAUSED(13), got instance={instance!r} route={route!r}") + if instance.get("functionProxyID") != "InstanceManagerOwner": + raise AssertionError(f"PAUSED instance owner is not InstanceManagerOwner: {instance!r}") + if route.get("functionProxyID") != "InstanceManagerOwner": + raise AssertionError(f"PAUSED route owner is not InstanceManagerOwner: {route!r}") + physical_fields = ( + "runtimeID", + "runtimeAddress", + "functionAgentID", + "containerID", + "containerIP", + "unitID", + "proxyGrpcAddress", + ) + for record_name, record in (("instance", instance), ("route", route)): + stale = {field: record.get(field) for field in physical_fields if record.get(field)} + if stale: + raise AssertionError(f"PAUSED {record_name} retains physical identity: {stale!r}") + if "portForward" in instance.get("extensions", {}): + raise AssertionError("PAUSED instance retains the source portForward extension") + snapshot = instance.get("snapshotInfo", {}) + if snapshot.get("status") not in (1, "SNAPSHOT_READY"): + raise AssertionError(f"PAUSED snapshot is not READY: {snapshot!r}") + if snapshot.get("checkpointID") != snapshot_id: + raise AssertionError( + f"PAUSED snapshot id mismatch: expected {snapshot_id!r}, got {snapshot!r}" + ) + for field in ("checkpointID", "storage", "sha256"): + if not snapshot.get(field): + raise AssertionError(f"PAUSED READY SnapshotInfo is missing {field}: {snapshot!r}") + if int(snapshot.get("size", 0)) <= 0: + raise AssertionError(f"PAUSED READY SnapshotInfo has no size: {snapshot!r}") + for field in ("instanceID", "requestID", "tenantID", "function"): + if instance.get(field) != source.get(field): + raise AssertionError(f"PAUSED changed logical field {field}") + if int(instance.get("version", 0)) <= int(source.get("version", 0)): + raise AssertionError("PAUSED version did not advance through CAS") + + +def _port_mappings_from_instance(instance: dict[str, Any]) -> dict[str, int]: + encoded = instance.get("extensions", {}).get("portForward") + if not encoded: + return {} + entries = json.loads(encoded) + mappings: dict[str, int] = {} + for entry in entries: + parts = str(entry).split(":") + if len(parts) != 3: + raise AssertionError(f"invalid canonical portForward entry: {entry!r}") + mappings[str(int(parts[2]))] = int(parts[1]) + return mappings + + +def _port_mappings_from_sandboxd(sandbox: dict[str, Any] | None) -> dict[str, int]: + if sandbox is None: + return {} + mappings: dict[str, int] = {} + for entry in sandbox.get("ports", []): + parts = str(entry).split(":") + if len(parts) != 3: + raise AssertionError(f"invalid sandboxd physical port fact: {entry!r}") + mappings[str(int(parts[2]))] = int(parts[1]) + return mappings + + +def _normalize_port_mappings(mappings: Any) -> dict[str, int]: + return {str(int(container)): int(host) for container, host in dict(mappings).items()} + + +def _inspect_sandboxd( + node_container: str, runtime_id: str, evidence_path: Path +) -> dict[str, Any] | None: + completed = _docker_exec( + node_container, "/usr/local/bin/sbox", "inspect", runtime_id, check=False + ) + if completed.returncode != 0: + raise RuntimeError( + f"sandboxd inspect failed for {runtime_id}: {completed.stderr or completed.stdout}" + ) + stdout = completed.stdout.strip() + if not stdout: + evidence = { + "sandboxId": runtime_id, + "found": False, + "stderr": completed.stderr.strip(), + } + evidence_path.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n") + return None + sandbox = json.loads(stdout) + evidence_path.write_text(json.dumps(sandbox, indent=2, sort_keys=True) + "\n") + return sandbox + + +def _function_proxy_pid(node_container: str) -> int: + result = _docker_exec(node_container, "pidof", "function_proxy", check=False) + pids = [int(value) for value in result.stdout.split() if value.isdigit()] + if not pids: + raise RuntimeError("function_proxy process is not running") + return min(pids) + + +def _function_proxy_registration_count(node_container: str) -> int: + result = _docker_exec( + node_container, + "bash", + "-lc", + "grep -ah 'succeed to register to local scheduler' " + "/home/yuanrong/logs/*function_proxy*.log 2>/dev/null | wc -l", + ) + return int(result.stdout.strip()) + + +def _restart_function_proxy( + node_container: str, source_pid: int, source_registration_count: int, timeout: int +) -> int: + _docker_exec(node_container, "kill", "-TERM", str(source_pid)) + deadline = time.monotonic() + timeout + target_pid = 0 + while time.monotonic() < deadline: + try: + observed = _function_proxy_pid(node_container) + except RuntimeError: + observed = 0 + if observed and observed != source_pid: + target_pid = observed + if _function_proxy_registration_count(node_container) > source_registration_count: + return target_pid + time.sleep(0.25) + raise TimeoutError( + f"FunctionProxy did not restart and re-register within {timeout}s " + f"(source={source_pid}, last={target_pid})" + ) + + +def _assert_snapshot_cleaned(authority: dict[str, Any], snapshot_id: str) -> None: + instance = authority["instance"]["value"] + snapshot = instance.get("snapshotInfo", {}) + if snapshot.get("checkpointID") == snapshot_id or snapshot.get("status") in ( + 1, + "SNAPSHOT_READY", + ): + raise AssertionError(f"exact READY snapshot was not cleaned after resume: {snapshot!r}") + for record in authority.get("matchingRecords", []): + if snapshot_id and snapshot_id in json.dumps(record.get("value", {}), sort_keys=True): + raise AssertionError(f"snapshot {snapshot_id} remains in ETCD winner state") + + +def _verify_cleanup( + node_container: str, + sandbox_id: str, + winner_runtime_id: str, + evidence_dir: Path, + timeout: int = 60, +) -> dict[str, Any]: + deadline = time.monotonic() + timeout + last: dict[str, Any] = {} + while time.monotonic() < deadline: + last = _capture_authority(node_container, sandbox_id, "after-delete", evidence_dir) + if last["instance"] is None and last["route"] is None: + physical = _inspect_sandboxd( + node_container, + winner_runtime_id, + evidence_dir / "sandboxd-winner-after-delete.json", + ) + if physical is None: + return last + time.sleep(0.25) + raise TimeoutError(f"delete cleanup did not converge: {last!r}") + + +def _fetch_public_once(url: str) -> str: + """Issue exactly one public request; resume convergence must not need retries.""" + with urllib.request.urlopen(url, timeout=10) as response: + return response.read().decode() + + +def _assert_public_paused_once(url: str) -> None: + """Require one authoritative PAUSED response from SandboxRouter.""" + try: + _fetch_public_once(url) + except urllib.error.HTTPError as exc: + body = exc.read().decode(errors="replace").lower() + if exc.code != 409 or "paused" not in body: + raise AssertionError( + f"expected HTTP 409 instance paused, got HTTP {exc.code}: {body!r}" + ) from exc + return + raise AssertionError("expected SandboxRouter to reject the PAUSED sandbox") + + +def _assert_public_missing_once(url: str) -> None: + """Require one authoritative missing response after exact cleanup.""" + try: + _fetch_public_once(url) + except urllib.error.HTTPError as exc: + if exc.code != 404: + body = exc.read().decode(errors="replace") + raise AssertionError(f"expected HTTP 404 after delete, got {exc.code}: {body!r}") from exc + return + raise AssertionError("expected SandboxRouter route to be absent after delete") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/standalone/pause_resume_function_matrix_e2e.py b/deploy/standalone/pause_resume_function_matrix_e2e.py new file mode 100644 index 0000000..4d8d248 --- /dev/null +++ b/deploy/standalone/pause_resume_function_matrix_e2e.py @@ -0,0 +1,876 @@ +#!/usr/bin/env python3 +"""Run the standalone RRT pause/resume functional matrix with JSON evidence.""" + +from __future__ import annotations + +import argparse +import json +import os +import tempfile +import threading +import time +import traceback +from dataclasses import asdict +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Callable + +import pause_resume_e2e as base +import pause_resume_stress_e2e as stress + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--report", type=Path, required=True) + parser.add_argument("--evidence-dir", type=Path, required=True) + parser.add_argument("--node-container", default="akernel-node") + parser.add_argument("--cpu", type=int, default=1000) + parser.add_argument("--memory", type=int, default=4096) + parser.add_argument("--create-timeout", type=int, default=240) + parser.add_argument( + "--host-checkpoint-root", + type=Path, + default=Path(__file__).resolve().parent + / "data" + / "sandboxd" + / "root" + / "checkpoints", + ) + args = parser.parse_args() + for name in ("YR_SERVER_ADDRESS", "YR_GATEWAY_ADDRESS", "YR_TOKEN"): + if not os.environ.get(name, "").strip(): + parser.error(f"{name} is required") + + from yr_sandbox import Sandbox + + started = time.time() + args.evidence_dir.mkdir(parents=True, exist_ok=True) + report: dict[str, Any] = { + "schemaVersion": 1, + "result": "failed", + "runtime": "RRT/Rust Runtime", + "sandboxBackend": "runsc", + "publicDataPlane": "SandboxRouter", + "startedAtUnix": int(started), + "matrix": { + "sdk-baseline": [], + "pause-resume-continuity": [], + "pause-delete-without-resume": [], + }, + } + managed: list[Any] = [] + watch_process, watch_stream = base._start_etcd_watch( + args.node_container, args.evidence_dir / "etcd-watch.jsonl" + ) + upstream_server, upstream_thread, upstream_url = _start_upstream_server() + + def step(group: str, name: str, operation: Callable[[], Any]) -> Any: + step_started = time.perf_counter() + try: + value = operation() + except Exception as exc: + report["matrix"][group].append( + { + "name": name, + "result": "failed", + "durationSeconds": round(time.perf_counter() - step_started, 6), + "errorType": type(exc).__name__, + "error": str(exc), + } + ) + raise + report["matrix"][group].append( + { + "name": name, + "result": "passed", + "durationSeconds": round(time.perf_counter() - step_started, 6), + } + ) + print(f"[PASS] {group}: {name}", flush=True) + return value + + baseline_views = stress._checkpoint_views( + args.node_container, args.host_checkpoint_root + ) + stress._assert_checkpoint_views_agree(baseline_views) + report["checkpointBaseline"] = baseline_views + report["containerCheckpointImages"] = baseline_views["container"] + report["hostCheckpointImages"] = baseline_views["host"] + idle_resources = stress._resource_view(args.node_container) + capacity = idle_resources["capacity"] + stress._assert_allocatable(idle_resources, capacity["CPU"], capacity["Memory"]) + report["idleResourceView"] = idle_resources + + primary: Any = None + primary_deleted = False + delete_case: Any = None + delete_case_deleted = False + primary_public_url = "" + delete_public_url = "" + primary_winner_container = "" + + try: + stamp = int(started) + primary_port = 18200 + primary_body = "akernel-function-matrix-public-ok" + primary = step( + "sdk-baseline", + "create RRT sandbox", + lambda: Sandbox( + name=f"akernel-functional-{stamp}", + runtime="runsc", + cpu=args.cpu, + memory=args.memory, + create_timeout=args.create_timeout, + schedule_timeout=180, + port_forwardings=[primary_port], + upstream=upstream_url, + ), + ) + managed.append(primary) + report["primarySandboxId"] = primary.id + step( + "sdk-baseline", + "sandbox get_info and is_running", + lambda: _assert_sandbox_running(primary), + ) + running_resources, running_convergence = stress._await_resources( + args.node_container, + capacity["CPU"] - args.cpu, + capacity["Memory"] - args.memory, + ) + report["primaryRunningResourceView"] = running_resources + report["primaryRunningResourceConvergenceSeconds"] = running_convergence + + step( + "sdk-baseline", + "foreground command stdout stderr exit", + lambda: _assert_foreground_command(primary), + ) + step( + "sdk-baseline", + "command environment and working directory", + lambda: _assert_command_environment_and_cwd(primary), + ) + step( + "sdk-baseline", + "background process list and kill", + lambda: _assert_background_process(primary), + ) + step( + "sdk-baseline", + "filesystem text binary metadata rename copy remove", + lambda: _assert_filesystem_matrix(primary), + ) + step( + "sdk-baseline", + "PTY input resize and independent sessions", + lambda: _assert_pty_matrix(primary), + ) + step( + "sdk-baseline", + "reverse tunnel request", + lambda: _assert_command_stdout( + primary.commands.run(f"curl -fsS {primary.get_tunnel_url()}"), + _UpstreamHandler.response_body.decode(), + ), + ) + + marker_path = "/tmp/akernel-functional-marker" + marker_value = "functional-continuity" + binary_path = "/tmp/akernel-functional-binary" + binary_value = bytes(range(256)) * 4 + primary.files.write(marker_path, marker_value) + primary.files.write(binary_path, binary_value) + stdin_handle = primary.commands.run( + "read value; printf 'continued:%s' \"$value\"", + background=True, + stdin=True, + ) + memory_handle, memory_before = step( + "sdk-baseline", + "start and mutate 32 MiB anonymous memory workload", + lambda: _start_memory_state_process(primary), + ) + report["primaryMemoryBeforePause"] = memory_before + _start_public_server(primary, primary_port, primary_body) + primary_public_url = primary.get_port_url(primary_port) + step( + "sdk-baseline", + "SandboxRouter public port before pause", + lambda: base._assert_equal( + base._fetch_public_once(primary_public_url), primary_body + ), + ) + source = base._capture_authority( + args.node_container, + primary.id, + "functional-source-running", + args.evidence_dir, + ) + base._assert_running_authority(source) + source_instance = source["instance"]["value"] + source_container = source_instance["containerID"] + report["primarySource"] = { + "containerId": source_container, + "runtimeId": source_instance["runtimeID"], + "functionProxyId": source_instance["functionProxyID"], + "portMappings": base._port_mappings_from_instance(source_instance), + } + + pause_result = step( + "pause-resume-continuity", + "pause sandbox", + lambda: primary.pause(ttl_seconds=1800), + ) + report["primaryPause"] = asdict(pause_result) + paused = base._capture_authority( + args.node_container, + primary.id, + "functional-paused", + args.evidence_dir, + ) + step( + "pause-resume-continuity", + "PAUSED authority owner snapshot route and cleared identity", + lambda: base._assert_paused_authority( + paused, pause_result.snapshot_id, source_instance + ), + ) + step( + "pause-resume-continuity", + "source sandboxd physical fact released", + lambda: base._assert_equal( + base._inspect_sandboxd( + args.node_container, + source_container, + args.evidence_dir / "sandboxd-functional-source-after-pause.json", + ), + None, + ), + ) + paused_resources, paused_resource_convergence = stress._await_resources( + args.node_container, capacity["CPU"], capacity["Memory"] + ) + report["primaryPausedResourceView"] = paused_resources + report["primaryPausedResourceConvergenceSeconds"] = paused_resource_convergence + paused_views = stress._checkpoint_views( + args.node_container, args.host_checkpoint_root, primary.id + ) + report["primaryCheckpointWhilePaused"] = paused_views + step( + "pause-resume-continuity", + "PAUSED data plane rejects file exec PTY and public port", + lambda: _assert_paused_data_plane(primary, primary_public_url), + ) + + resume_result = step( + "pause-resume-continuity", "resume sandbox", primary.resume + ) + report["primaryResume"] = asdict(resume_result) + immediate = step( + "pause-resume-continuity", + "first file exec PTY and public requests after resume", + lambda: _assert_first_requests_after_resume( + primary, + marker_path, + marker_value, + primary_public_url, + primary_body, + ), + ) + report["firstRequestsAfterResumeSeconds"] = immediate + step( + "pause-resume-continuity", + "binary file continuity", + lambda: base._assert_equal( + primary.files.read(binary_path, format="bytes"), binary_value + ), + ) + step( + "pause-resume-continuity", + "reverse tunnel after resume", + lambda: _assert_command_stdout( + primary.commands.run(f"curl -fsS {primary.get_tunnel_url()}"), + _UpstreamHandler.response_body.decode(), + ), + ) + step( + "pause-resume-continuity", + "stdin-blocked process continuity", + lambda: _complete_stdin_process(stdin_handle), + ) + memory_after = step( + "pause-resume-continuity", + "anonymous memory PID counter checksum and sentinel continuity", + lambda: _complete_memory_state_process( + primary, memory_handle, memory_before + ), + ) + report["primaryMemoryAfterResume"] = memory_after + winner = base._capture_authority( + args.node_container, + primary.id, + "functional-winner-running", + args.evidence_dir, + ) + step( + "pause-resume-continuity", + "RUNNING winner and exact snapshot cleanup", + lambda: _assert_winner( + args.node_container, + args.evidence_dir, + winner, + resume_result, + pause_result.snapshot_id, + ), + ) + winner_instance = winner["instance"]["value"] + primary_winner_container = winner_instance["containerID"] + report["primaryWinner"] = { + "containerId": primary_winner_container, + "runtimeId": winner_instance["runtimeID"], + "functionProxyId": winner_instance["functionProxyID"], + "portMappings": base._port_mappings_from_instance(winner_instance), + "sourcePortsMayBeReused": True, + } + resumed_resources, resumed_resource_convergence = stress._await_resources( + args.node_container, + capacity["CPU"] - args.cpu, + capacity["Memory"] - args.memory, + ) + report["primaryResumedResourceView"] = resumed_resources + report["primaryResumedResourceConvergenceSeconds"] = resumed_resource_convergence + resumed_views = stress._checkpoint_views( + args.node_container, args.host_checkpoint_root, primary.id + ) + if resumed_views["container"]: + raise AssertionError( + f"checkpoint.img remains after committed resume: {resumed_views!r}" + ) + report["primaryCheckpointAfterResume"] = resumed_views + + step("pause-resume-continuity", "delete resumed sandbox", primary.kill) + managed.remove(primary) + primary_deleted = True + step( + "pause-resume-continuity", + "resumed sandbox ETCD sandboxd and public route cleanup", + lambda: _assert_deleted( + args, + primary.id, + primary_winner_container, + primary_public_url, + "primary-after-delete", + ), + ) + stress._await_resources( + args.node_container, capacity["CPU"], capacity["Memory"] + ) + + delete_port = 18201 + delete_body = "akernel-pause-delete-public-ok" + delete_case = step( + "pause-delete-without-resume", + "create delete-case RRT sandbox", + lambda: Sandbox( + name=f"akernel-pause-delete-{stamp}", + runtime="runsc", + cpu=args.cpu, + memory=args.memory, + create_timeout=args.create_timeout, + schedule_timeout=180, + port_forwardings=[delete_port], + ), + ) + managed.append(delete_case) + _start_public_server(delete_case, delete_port, delete_body) + delete_public_url = delete_case.get_port_url(delete_port) + base._assert_equal(base._fetch_public_once(delete_public_url), delete_body) + delete_source = base._capture_authority( + args.node_container, + delete_case.id, + "pause-delete-source-running", + args.evidence_dir, + ) + base._assert_running_authority(delete_source) + delete_source_instance = delete_source["instance"]["value"] + delete_source_container = delete_source_instance["containerID"] + delete_pause = step( + "pause-delete-without-resume", + "pause delete-case sandbox", + lambda: delete_case.pause(ttl_seconds=1800), + ) + report["pauseDeletePause"] = asdict(delete_pause) + delete_paused = base._capture_authority( + args.node_container, + delete_case.id, + "pause-delete-paused", + args.evidence_dir, + ) + step( + "pause-delete-without-resume", + "verify delete-case PAUSED authority", + lambda: base._assert_paused_authority( + delete_paused, delete_pause.snapshot_id, delete_source_instance + ), + ) + report["pauseDeleteCheckpointWhilePaused"] = stress._checkpoint_views( + args.node_container, args.host_checkpoint_root, delete_case.id + ) + step( + "pause-delete-without-resume", + "delete directly from PAUSED", + delete_case.kill, + ) + managed.remove(delete_case) + delete_case_deleted = True + step( + "pause-delete-without-resume", + "PAUSED delete cleans ETCD sandboxd route snapshot and checkpoint", + lambda: _assert_deleted( + args, + delete_case.id, + delete_source_container, + delete_public_url, + "pause-delete-after-delete", + ), + ) + final_resources, final_resource_convergence = stress._await_resources( + args.node_container, capacity["CPU"], capacity["Memory"] + ) + report["finalResourceView"] = final_resources + report["finalResourceViewConvergenceSeconds"] = final_resource_convergence + final_views = stress._checkpoint_views( + args.node_container, args.host_checkpoint_root + ) + stress._assert_checkpoint_views_agree(final_views) + report["checkpointFinal"] = final_views + report["containerCheckpointImages"] = final_views["container"] + report["hostCheckpointImages"] = final_views["host"] + _assert_no_new_checkpoint_images(baseline_views, final_views) + report["result"] = "passed" + except Exception as exc: + report["errorType"] = type(exc).__name__ + report["error"] = str(exc) + report["traceback"] = traceback.format_exc() + finally: + for sandbox in reversed(managed): + try: + sandbox.kill() + except Exception: + pass + if primary is not None and not primary_deleted: + report["primaryCleanupAttempted"] = True + if delete_case is not None and not delete_case_deleted: + report["pauseDeleteCleanupAttempted"] = True + upstream_server.shutdown() + upstream_server.server_close() + upstream_thread.join(timeout=5) + base._stop_etcd_watch(watch_process, watch_stream) + report["finishedAtUnix"] = int(time.time()) + report["durationSeconds"] = round(time.time() - started, 6) + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + + if report["result"] != "passed": + print(json.dumps({"result": "failed", "error": report.get("error")}), flush=True) + return 1 + print(json.dumps({"result": "passed", "report": str(args.report)}), flush=True) + return 0 + + +class _UpstreamHandler(BaseHTTPRequestHandler): + response_body = b"akernel-reverse-tunnel-ok" + + def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + self.send_response(200) + self.send_header("Content-Length", str(len(self.response_body))) + self.end_headers() + self.wfile.write(self.response_body) + + def log_message(self, _format: str, *_args: Any) -> None: + return + + +def _start_upstream_server() -> tuple[ThreadingHTTPServer, threading.Thread, str]: + server = ThreadingHTTPServer(("127.0.0.1", 0), _UpstreamHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + address, port = server.server_address + return server, thread, f"http://{address}:{port}" + + +def _assert_command_stdout(result: Any, expected: str) -> None: + if result.exit_code != 0 or result.stdout != expected: + raise AssertionError( + f"expected rc=0 stdout={expected!r}, got " + f"rc={result.exit_code} stdout={result.stdout!r} stderr={result.stderr!r}" + ) + + +def _assert_sandbox_running(sandbox: Any) -> None: + info = sandbox.get_info() + if info.state != "running" or not sandbox.is_running(): + raise AssertionError(f"sandbox is not running: {info!r}") + + +def _assert_foreground_command(sandbox: Any) -> None: + result = sandbox.commands.run("printf stdout-ok; printf stderr-ok >&2; exit 7") + if result.exit_code != 7 or result.stdout != "stdout-ok" or result.stderr != "stderr-ok": + raise AssertionError(f"foreground command result mismatch: {result!r}") + + +def _assert_command_environment_and_cwd(sandbox: Any) -> None: + sandbox.files.make_dir("/tmp/akernel-functional-cwd") + result = sandbox.commands.run( + "printf '%s:%s' \"$FUNCTION_MATRIX_ENV\" \"$PWD\"", + envs={"FUNCTION_MATRIX_ENV": "env-ok"}, + cwd="/tmp/akernel-functional-cwd", + ) + _assert_command_stdout(result, "env-ok:/tmp/akernel-functional-cwd") + + +def _assert_background_process(sandbox: Any) -> None: + handle = sandbox.commands.run("sleep 300", background=True) + processes = sandbox.commands.list() + match = next((item for item in processes if item.pid == handle.pid), None) + if match is None or not match.running or "sleep 300" not in match.command: + raise AssertionError(f"background process missing from list: {processes!r}") + if not handle.kill(): + raise AssertionError("background process kill returned false") + + +def _assert_filesystem_matrix(sandbox: Any) -> None: + root = "/tmp/akernel-functional-files" + sandbox.files.make_dir(root) + text_path = f"{root}/text.txt" + binary_path = f"{root}/binary.bin" + binary = bytes(range(256)) * 8 + sandbox.files.write(text_path, "text-ok") + sandbox.files.write(binary_path, binary) + base._assert_equal(sandbox.files.read(text_path), "text-ok") + base._assert_equal(sandbox.files.read(binary_path, format="bytes"), binary) + if not sandbox.files.exists(text_path): + raise AssertionError("written text file does not exist") + info = sandbox.files.get_info(binary_path) + if info.type != "file" or info.size != len(binary): + raise AssertionError(f"binary metadata mismatch: {info!r}") + entries = sandbox.files.list(root, depth=2) + if {entry.name for entry in entries}.isdisjoint({"text.txt", "binary.bin"}): + raise AssertionError(f"filesystem list omitted files: {entries!r}") + renamed_path = f"{root}/renamed.txt" + sandbox.files.rename(text_path, renamed_path) + if sandbox.files.exists(text_path) or not sandbox.files.exists(renamed_path): + raise AssertionError("filesystem rename did not update existence") + + with tempfile.TemporaryDirectory() as temporary: + local_root = Path(temporary) + upload = local_root / "upload.bin" + upload.write_bytes(b"copy-from-local\x00ok") + remote_upload = f"{root}/upload.bin" + sandbox.files.copy_from_local(str(upload), remote_upload) + base._assert_equal( + sandbox.files.read(remote_upload, format="bytes"), upload.read_bytes() + ) + download = local_root / "download.bin" + sandbox.files.copy_to_local(remote_upload, str(download)) + base._assert_equal(download.read_bytes(), upload.read_bytes()) + + directory = local_root / "tree" + directory.mkdir() + (directory / "nested.txt").write_text("directory-copy-ok") + remote_directory = f"{root}/tree" + sandbox.files.copy_from_local(str(directory), remote_directory) + base._assert_equal( + sandbox.files.read(f"{remote_directory}/nested.txt"), + "directory-copy-ok", + ) + downloaded_directory = local_root / "downloaded-tree" + sandbox.files.copy_to_local(remote_directory, str(downloaded_directory)) + base._assert_equal( + (downloaded_directory / "nested.txt").read_text(), + "directory-copy-ok", + ) + + sandbox.files.remove(renamed_path) + if sandbox.files.exists(renamed_path): + raise AssertionError("filesystem remove left the renamed file") + + +def _run_pty(sandbox: Any, marker: bytes, exit_code: int) -> bytes: + output = bytearray() + with sandbox.pty.create(on_data=output.extend) as session: + session.resize(rows=41, cols=121) + session.send_stdin(b"printf '" + marker + b"\\n'\n") + session.send_stdin(f"exit {exit_code}\n".encode()) + observed = session.wait(timeout=30) + if observed != exit_code: + raise AssertionError(f"PTY exit mismatch: expected {exit_code}, got {observed}") + if marker not in output: + raise AssertionError(f"PTY output omitted {marker!r}: {bytes(output)!r}") + return bytes(output) + + +def _assert_pty_matrix(sandbox: Any) -> None: + _run_pty(sandbox, b"PTY_BASELINE", 3) + first_output = bytearray() + second_output = bytearray() + with ( + sandbox.pty.create(on_data=first_output.extend) as first, + sandbox.pty.create(on_data=second_output.extend) as second, + ): + first.send_stdin(b"printf 'PTY_FIRST\\n'\nexit 4\n") + second.send_stdin(b"printf 'PTY_SECOND\\n'\nexit 5\n") + if first.wait(timeout=30) != 4 or second.wait(timeout=30) != 5: + raise AssertionError("independent PTY exit codes mismatch") + if b"PTY_FIRST" not in first_output or b"PTY_SECOND" in first_output: + raise AssertionError(f"first PTY output leaked: {bytes(first_output)!r}") + if b"PTY_SECOND" not in second_output or b"PTY_FIRST" in second_output: + raise AssertionError(f"second PTY output leaked: {bytes(second_output)!r}") + + +def _start_public_server(sandbox: Any, port: int, body: str) -> None: + ready = f"/tmp/akernel-functional-public-{port}.ready" + command = rf'''perl -MSocket -e '$|=1; socket(S,PF_INET,SOCK_STREAM,getprotobyname("tcp")); setsockopt(S,SOL_SOCKET,SO_REUSEADDR,1); bind(S,sockaddr_in({port},INADDR_ANY)) or die $!; listen(S,10); open(F, ">", "{ready}") or die $!; print F "ready"; close F; while(accept(C,S)){{ print C "HTTP/1.1 200 OK\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n{body}"; close C; }}' ''' + sandbox.commands.run(command, background=True) + result = sandbox.commands.run( + f"while [ ! -f {ready} ]; do sleep 0.05; done; printf ready" + ) + _assert_command_stdout(result, "ready") + + +def _assert_paused_error(operation: Callable[[], Any], name: str) -> None: + try: + operation() + except Exception as exc: + message = str(exc).lower() + if "paused" in message or "409" in message: + return + raise AssertionError(f"{name} did not return PAUSED: {type(exc).__name__}: {exc}") from exc + raise AssertionError(f"{name} unexpectedly succeeded while PAUSED") + + +def _assert_paused_data_plane(sandbox: Any, public_url: str) -> None: + _assert_paused_error(lambda: sandbox.files.read("/tmp/akernel-functional-marker"), "file") + _assert_paused_error(lambda: sandbox.commands.run("printf forbidden"), "exec") + _assert_paused_error( + lambda: sandbox.pty.create(command=["/bin/sh", "-c", "exit 0"], timeout=10), + "PTY", + ) + base._assert_public_paused_once(public_url) + + +def _timed(operation: Callable[[], Any]) -> tuple[Any, float]: + started = time.perf_counter() + value = operation() + return value, round(time.perf_counter() - started, 6) + + +def _assert_first_requests_after_resume( + sandbox: Any, + marker_path: str, + marker_value: str, + public_url: str, + public_body: str, +) -> dict[str, float]: + marker, file_seconds = _timed(lambda: sandbox.files.read(marker_path)) + base._assert_equal(marker, marker_value) + command, exec_seconds = _timed(lambda: sandbox.commands.run("printf immediate-exec-ok")) + _assert_command_stdout(command, "immediate-exec-ok") + _, pty_seconds = _timed(lambda: _run_pty(sandbox, b"PTY_AFTER_RESUME", 6)) + body, public_seconds = _timed(lambda: base._fetch_public_once(public_url)) + base._assert_equal(body, public_body) + return { + "file": file_seconds, + "exec": exec_seconds, + "pty": pty_seconds, + "public": public_seconds, + } + + +def _complete_stdin_process(handle: Any) -> None: + handle.send_stdin("matrix\n", eof=True) + _assert_command_stdout(handle.wait(timeout=90), "continued:matrix") + + +_MEMORY_STATE_SCRIPT = r'''#!/usr/bin/perl +use strict; +use warnings; + +my $state_file = "/tmp/akernel-memory-state.tsv"; +my $ready_file = "/tmp/akernel-memory-state.ready"; +my $bytes = 32 * 1024 * 1024; +my $offset = 17 * 1024 * 1024 + 19; +my $blob = "0123456789abcdef" x ($bytes / 16); +my $counter = 41; +my $sequence = 0; + +sub publish_state { + my $temporary = "$state_file.tmp"; + open(my $output, ">", $temporary) or die "open state: $!"; + my $checksum = unpack("%32C*", $blob); + my $sentinel = ord(substr($blob, $offset, 1)); + print $output join("\t", $sequence, $$, $counter, length($blob), $checksum, $sentinel), "\n"; + close($output) or die "close state: $!"; + rename($temporary, $state_file) or die "rename state: $!"; +} + +open(my $ready, ">", $ready_file) or die "open ready: $!"; +print $ready "ready\n"; +close($ready) or die "close ready: $!"; + +while (my $line = ) { + chomp($line); + last if $line eq "quit"; + if ($line eq "mutate") { + $counter += 1; + substr($blob, $offset, 1) = chr(90); + } elsif ($line ne "report") { + die "unknown command: $line"; + } + $sequence += 1; + publish_state(); +} +''' + + +def _parse_memory_state(payload: str) -> dict[str, int]: + fields = payload.strip().split("\t") + if len(fields) != 6: + raise AssertionError(f"invalid memory state payload: {payload!r}") + values = [int(field) for field in fields] + return { + "sequence": values[0], + "pid": values[1], + "counter": values[2], + "length": values[3], + "checksum": values[4], + "sentinelByte": values[5], + } + + +def _await_memory_state(sandbox: Any, sequence: int) -> dict[str, int]: + state_path = "/tmp/akernel-memory-state.tsv" + command = sandbox.commands.run( + "i=0; " + f"while [ ! -f {state_path} ] || " + f"[ \"$(cut -f1 {state_path} 2>/dev/null)\" != \"{sequence}\" ]; do " + "i=$((i + 1)); [ \"$i\" -ge 600 ] && exit 124; sleep 0.05; done; " + "printf ready" + ) + _assert_command_stdout(command, "ready") + return _parse_memory_state(sandbox.files.read(state_path)) + + +def _start_memory_state_process(sandbox: Any) -> tuple[Any, dict[str, int]]: + script_path = "/tmp/akernel-memory-state.pl" + sandbox.files.write(script_path, _MEMORY_STATE_SCRIPT) + handle = sandbox.commands.run( + f"perl {script_path}", background=True, stdin=True + ) + ready = sandbox.commands.run( + "i=0; while [ ! -f /tmp/akernel-memory-state.ready ]; do " + "i=$((i + 1)); [ \"$i\" -ge 600 ] && exit 124; sleep 0.05; done; printf ready" + ) + _assert_command_stdout(ready, "ready") + handle.send_stdin("mutate\n") + state = _await_memory_state(sandbox, 1) + expected = { + "sequence": 1, + "counter": 42, + "length": 32 * 1024 * 1024, + "sentinelByte": 90, + } + for key, value in expected.items(): + if state[key] != value: + raise AssertionError( + f"memory workload initialization mismatch for {key}: {state!r}" + ) + return handle, state + + +def _assert_memory_state_continuity( + before: dict[str, int], after: dict[str, int] +) -> None: + expected_after = {**before, "sequence": before["sequence"] + 1} + if after != expected_after: + raise AssertionError( + "memory process identity/state changed across resume: " + f"before={before!r} after={after!r}" + ) + + +def _complete_memory_state_process( + sandbox: Any, handle: Any, before: dict[str, int] +) -> dict[str, int]: + handle.send_stdin("report\n") + after = _await_memory_state(sandbox, before["sequence"] + 1) + _assert_memory_state_continuity(before, after) + handle.send_stdin("quit\n", eof=True) + _assert_command_stdout(handle.wait(timeout=90), "") + return after + + +def _assert_winner( + node_container: str, + evidence_dir: Path, + winner: dict[str, Any], + resume_result: Any, + snapshot_id: str, +) -> None: + base._assert_running_authority(winner) + base._assert_snapshot_cleaned(winner, snapshot_id) + instance = winner["instance"]["value"] + mappings = base._port_mappings_from_instance(instance) + base._assert_equal(base._normalize_port_mappings(resume_result.port_mappings), mappings) + physical = base._inspect_sandboxd( + node_container, + instance["containerID"], + evidence_dir / "sandboxd-functional-winner-running.json", + ) + if physical is None: + raise AssertionError("sandboxd has no winner physical fact") + base._assert_equal(base._port_mappings_from_sandboxd(physical), mappings) + + +def _assert_deleted( + args: argparse.Namespace, + sandbox_id: str, + physical_id: str, + public_url: str, + phase: str, +) -> None: + cleanup_dir = args.evidence_dir / phase + cleanup_dir.mkdir(parents=True, exist_ok=True) + base._verify_cleanup( + args.node_container, + sandbox_id, + physical_id, + cleanup_dir, + ) + base._assert_public_missing_once(public_url) + views = stress._checkpoint_views( + args.node_container, args.host_checkpoint_root, sandbox_id + ) + if views["container"]: + raise AssertionError(f"deleted sandbox retains checkpoint.img: {views!r}") + + +def _assert_no_new_checkpoint_images( + baseline: dict[str, list[dict[str, Any]]], + final: dict[str, list[dict[str, Any]]], +) -> None: + def facts(views: dict[str, list[dict[str, Any]]]) -> set[tuple[str, int]]: + return { + (str(item["relativePath"]), int(item["size"])) + for item in views["container"] + } + + new_facts = facts(final) - facts(baseline) + if new_facts: + raise AssertionError(f"new checkpoint.img files remain: {sorted(new_facts)!r}") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/standalone/pause_resume_stress_e2e.py b/deploy/standalone/pause_resume_stress_e2e.py new file mode 100644 index 0000000..1035346 --- /dev/null +++ b/deploy/standalone/pause_resume_stress_e2e.py @@ -0,0 +1,594 @@ +#!/usr/bin/env python3 +"""Exercise repeated and interleaved RRT pause/resume with resource evidence.""" + +from __future__ import annotations + +import argparse +import json +import os +import statistics +import sys +import time +import traceback +from pathlib import Path +from typing import Any, Callable + +import pause_resume_e2e as base + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--report", type=Path, required=True) + parser.add_argument("--evidence-dir", type=Path, required=True) + parser.add_argument("--node-container", default="akernel-node") + parser.add_argument( + "--host-checkpoint-root", + type=Path, + default=Path(__file__).resolve().parent + / "data" + / "sandboxd" + / "root" + / "checkpoints", + ) + parser.add_argument("--loop-count", type=int, default=5) + parser.add_argument("--instance-count", type=int, default=3) + parser.add_argument("--cpu", type=int, default=1000) + parser.add_argument("--memory", type=int, default=4096) + parser.add_argument("--create-timeout", type=int, default=240) + args = parser.parse_args() + if args.loop_count < 2 or args.instance_count < 3: + parser.error("loop-count must be >=2 and instance-count must be >=3") + for name in ("YR_SERVER_ADDRESS", "YR_GATEWAY_ADDRESS", "YR_TOKEN"): + if not os.environ.get(name, "").strip(): + parser.error(f"{name} is required") + + from yr_sandbox import Sandbox + + args.evidence_dir.mkdir(parents=True, exist_ok=True) + report: dict[str, Any] = { + "schemaVersion": 1, + "result": "failed", + "runtime": "RRT/Rust Runtime", + "publicDataPlane": "SandboxRouter", + "startedAtUnix": int(time.time()), + "loopCount": args.loop_count, + "instanceCount": args.instance_count, + "single-instance-loop": {}, + "multi-instance-interleaved": {}, + } + managed: list[Any] = [] + baseline_checkpoint_views = _checkpoint_views( + args.node_container, args.host_checkpoint_root + ) + report["checkpointBaseline"] = baseline_checkpoint_views + report["containerCheckpointImages"] = baseline_checkpoint_views["container"] + report["hostCheckpointImages"] = baseline_checkpoint_views["host"] + + try: + idle = _resource_view(args.node_container) + capacity = idle["capacity"] + _assert_allocatable(idle, capacity["CPU"], capacity["Memory"]) + report["idleResourceView"] = idle + + single = _run_single_loop(Sandbox, args, capacity, managed) + report["single-instance-loop"] = single + + interleaved = _run_interleaved(Sandbox, args, capacity, managed) + report["multi-instance-interleaved"] = interleaved + + final_resources, convergence = _await_resources( + args.node_container, capacity["CPU"], capacity["Memory"] + ) + report["finalResourceView"] = final_resources + report["finalResourceViewConvergenceSeconds"] = convergence + final_checkpoint_views = _checkpoint_views( + args.node_container, args.host_checkpoint_root + ) + report["checkpointFinal"] = final_checkpoint_views + report["containerCheckpointImages"] = final_checkpoint_views["container"] + report["hostCheckpointImages"] = final_checkpoint_views["host"] + baseline_facts = { + (item["relativePath"], item["size"]) + for item in baseline_checkpoint_views["container"] + } + new_files = [ + item + for item in final_checkpoint_views["container"] + if (item["relativePath"], item["size"]) not in baseline_facts + ] + report["newCheckpointImages"] = new_files + if new_files: + raise AssertionError(f"new checkpoint.img cache files remain: {new_files!r}") + report["performanceSummary"] = _performance_summary(single, interleaved) + report["result"] = "passed" + except Exception as exc: + report["errorType"] = type(exc).__name__ + report["error"] = str(exc) + report["traceback"] = traceback.format_exc() + finally: + for sandbox in reversed(managed): + try: + sandbox.kill() + except Exception: + pass + report["finishedAtUnix"] = int(time.time()) + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + + if report["result"] != "passed": + print(json.dumps({"result": "failed", "error": report.get("error")}), file=sys.stderr) + return 1 + print(json.dumps({"result": "passed", "report": str(args.report)})) + return 0 + + +def _timed(operation: Callable[[], Any]) -> tuple[Any, float]: + started = time.perf_counter() + value = operation() + return value, round(time.perf_counter() - started, 6) + + +def _create_sandbox(Sandbox: Any, args: argparse.Namespace, name: str, port: int) -> Any: + return Sandbox( + name=name, + runtime="runsc", + cpu=args.cpu, + memory=args.memory, + create_timeout=args.create_timeout, + schedule_timeout=180, + port_forwardings=[port], + ) + + +def _start_workload(sandbox: Any, marker: str, port: int, body: str) -> str: + sandbox.files.write(marker, "initial") + sandbox.commands.run( + "sh -c 'echo $$ > /tmp/akernel-loop-process.pid; while :; do sleep 60; done'", + background=True, + ) + ready = f"/tmp/akernel-public-{port}-ready" + server = rf'''perl -MSocket -e '$|=1; socket(S,PF_INET,SOCK_STREAM,getprotobyname("tcp")); setsockopt(S,SOL_SOCKET,SO_REUSEADDR,1); bind(S,sockaddr_in({port},INADDR_ANY)) or die $!; listen(S,10); open(F, ">", "{ready}") or die $!; print F "ready"; close F; while(accept(C,S)){{ print C "HTTP/1.1 200 OK\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n{body}"; close C; }}' ''' + sandbox.commands.run(server, background=True) + result = sandbox.commands.run(f"while [ ! -f {ready} ]; do sleep 0.05; done; printf ready") + base._assert_command(result, "ready") + public_url = sandbox.get_port_url(port) + base._assert_equal(base._fetch_public_once(public_url), body) + return public_url + + +def _run_single_loop( + Sandbox: Any, + args: argparse.Namespace, + capacity: dict[str, int], + managed: list[Any], +) -> dict[str, Any]: + started = int(time.time()) + marker = "/tmp/akernel-loop-marker" + public_port = 18081 + public_body = "akernel-loop-ok" + sandbox, create_seconds = _timed( + lambda: _create_sandbox(Sandbox, args, f"akernel-loop-{started}", public_port) + ) + managed.append(sandbox) + public_url = _start_workload(sandbox, marker, public_port, public_body) + running_resources, create_convergence = _await_resources( + args.node_container, + capacity["CPU"] - args.cpu, + capacity["Memory"] - args.memory, + ) + result: dict[str, Any] = { + "sandboxId": sandbox.id, + "createSeconds": create_seconds, + "createResourceViewConvergenceSeconds": create_convergence, + "runningResourceView": running_resources, + "cycles": [], + } + + for cycle in range(1, args.loop_count + 1): + marker_value = f"loop-cycle-{cycle}" + sandbox.files.write(marker, marker_value) + source = base._capture_authority( + args.node_container, + sandbox.id, + f"loop-{cycle}-source-running", + args.evidence_dir, + ) + base._assert_running_authority(source) + + pause_result, pause_seconds = _timed(lambda: sandbox.pause(ttl_seconds=1800)) + paused = base._capture_authority( + args.node_container, sandbox.id, f"loop-{cycle}-paused", args.evidence_dir + ) + base._assert_paused_authority(paused, pause_result.snapshot_id, source["instance"]["value"]) + paused_resources, paused_convergence = _await_resources( + args.node_container, capacity["CPU"], capacity["Memory"] + ) + paused_images = _checkpoint_views( + args.node_container, args.host_checkpoint_root, sandbox.id + ) + base._assert_public_paused_once(public_url) + + resume_result, resume_seconds = _timed(sandbox.resume) + marker_after, first_file_seconds = _timed(lambda: sandbox.files.read(marker)) + base._assert_equal(marker_after, marker_value) + exec_after, first_exec_seconds = _timed( + lambda: sandbox.commands.run( + "kill -0 $(cat /tmp/akernel-loop-process.pid) && printf loop-running" + ) + ) + base._assert_command(exec_after, "loop-running") + public_after, first_public_seconds = _timed(lambda: base._fetch_public_once(public_url)) + base._assert_equal(public_after, public_body) + + winner = base._capture_authority( + args.node_container, + sandbox.id, + f"loop-{cycle}-winner-running", + args.evidence_dir, + ) + base._assert_running_authority(winner) + base._assert_snapshot_cleaned(winner, pause_result.snapshot_id) + winner_mappings = base._port_mappings_from_instance(winner["instance"]["value"]) + base._assert_equal(base._normalize_port_mappings(resume_result.port_mappings), winner_mappings) + running_resources, running_convergence = _await_resources( + args.node_container, + capacity["CPU"] - args.cpu, + capacity["Memory"] - args.memory, + ) + resumed_images = _checkpoint_views( + args.node_container, args.host_checkpoint_root, sandbox.id + ) + if resumed_images["container"]: + raise AssertionError( + f"cycle {cycle} retains checkpoint.img after successful resume: {resumed_images!r}" + ) + result["cycles"].append( + { + "cycle": cycle, + "snapshotId": pause_result.snapshot_id, + "pauseSeconds": pause_seconds, + "resumeSeconds": resume_seconds, + "firstFileSeconds": first_file_seconds, + "firstExecSeconds": first_exec_seconds, + "firstPublicSeconds": first_public_seconds, + "resourceViewConvergenceSeconds": { + "paused": paused_convergence, + "running": running_convergence, + }, + "pausedResourceView": paused_resources, + "runningResourceView": running_resources, + "checkpointImagesWhilePaused": paused_images, + "checkpointImagesAfterResume": resumed_images, + "portMappings": winner_mappings, + } + ) + + _, delete_seconds = _timed(sandbox.kill) + managed.remove(sandbox) + _, cleanup_convergence = _await_resources( + args.node_container, capacity["CPU"], capacity["Memory"] + ) + result["deleteSeconds"] = delete_seconds + result["cleanupResourceViewConvergenceSeconds"] = cleanup_convergence + result["checkpointImagesAfterDelete"] = _checkpoint_views( + args.node_container, args.host_checkpoint_root, sandbox.id + ) + if result["checkpointImagesAfterDelete"]["container"]: + raise AssertionError("single loop left checkpoint.img after delete") + return result + + +def _run_interleaved( + Sandbox: Any, + args: argparse.Namespace, + capacity: dict[str, int], + managed: list[Any], +) -> dict[str, Any]: + stamp = int(time.time()) + sandboxes: list[Any] = [] + metadata: list[dict[str, Any]] = [] + for index in range(args.instance_count): + port = 18100 + index + body = f"akernel-interleaved-{index}" + sandbox, create_seconds = _timed( + lambda index=index, port=port: _create_sandbox( + Sandbox, args, f"akernel-cross-{stamp}-{index}", port + ) + ) + sandboxes.append(sandbox) + managed.append(sandbox) + marker = f"/tmp/akernel-cross-{index}-marker" + public_url = _start_workload(sandbox, marker, port, body) + sandbox.files.write(marker, f"cross-{index}") + metadata.append( + { + "sandbox": sandbox, + "marker": marker, + "body": body, + "publicURL": public_url, + "createSeconds": create_seconds, + "state": "running", + } + ) + _await_resources( + args.node_container, + capacity["CPU"] - args.cpu * (index + 1), + capacity["Memory"] - args.memory * (index + 1), + ) + + actions = [ + ("pause", 0), + ("pause", 1), + ("resume", 0), + ("pause", 2), + ("resume", 1), + ("resume", 2), + ] + action_evidence: list[dict[str, Any]] = [] + for ordinal, (operation, index) in enumerate(actions, start=1): + item = metadata[index] + sandbox = item["sandbox"] + evidence: dict[str, Any] = {"ordinal": ordinal, "operation": operation, "index": index} + if operation == "pause": + source = base._capture_authority( + args.node_container, + sandbox.id, + f"cross-{ordinal}-{index}-source-running", + args.evidence_dir, + ) + pause_result, elapsed = _timed(lambda: sandbox.pause(ttl_seconds=1800)) + paused = base._capture_authority( + args.node_container, + sandbox.id, + f"cross-{ordinal}-{index}-paused", + args.evidence_dir, + ) + base._assert_paused_authority( + paused, pause_result.snapshot_id, source["instance"]["value"] + ) + base._assert_public_paused_once(item["publicURL"]) + item["state"] = "paused" + item["snapshotId"] = pause_result.snapshot_id + evidence["snapshotId"] = pause_result.snapshot_id + evidence["pauseSeconds"] = elapsed + evidence["checkpointImages"] = _checkpoint_views( + args.node_container, args.host_checkpoint_root, sandbox.id + ) + else: + resume_result, elapsed = _timed(sandbox.resume) + marker_value, first_file = _timed(lambda: sandbox.files.read(item["marker"])) + base._assert_equal(marker_value, f"cross-{index}") + exec_result, first_exec = _timed(lambda: sandbox.commands.run("printf cross-running")) + base._assert_command(exec_result, "cross-running") + public_body, first_public = _timed( + lambda: base._fetch_public_once(item["publicURL"]) + ) + base._assert_equal(public_body, item["body"]) + winner = base._capture_authority( + args.node_container, + sandbox.id, + f"cross-{ordinal}-{index}-winner-running", + args.evidence_dir, + ) + base._assert_running_authority(winner) + base._assert_snapshot_cleaned(winner, item["snapshotId"]) + base._assert_equal( + base._normalize_port_mappings(resume_result.port_mappings), + base._port_mappings_from_instance(winner["instance"]["value"]), + ) + item["state"] = "running" + evidence.update( + { + "resumeSeconds": elapsed, + "firstFileSeconds": first_file, + "firstExecSeconds": first_exec, + "firstPublicSeconds": first_public, + "checkpointImages": _checkpoint_views( + args.node_container, args.host_checkpoint_root, sandbox.id + ), + } + ) + if evidence["checkpointImages"]["container"]: + raise AssertionError( + f"interleaved resume {index} retains checkpoint.img: {evidence['checkpointImages']!r}" + ) + + running_count = sum(item["state"] == "running" for item in metadata) + resources, convergence = _await_resources( + args.node_container, + capacity["CPU"] - args.cpu * running_count, + capacity["Memory"] - args.memory * running_count, + ) + evidence["runningCount"] = running_count + evidence["resourceView"] = resources + evidence["resourceViewConvergenceSeconds"] = convergence + action_evidence.append(evidence) + + for index, item in enumerate(metadata): + sandbox = item["sandbox"] + base._assert_equal(sandbox.files.read(item["marker"]), f"cross-{index}") + sandbox.kill() + managed.remove(sandbox) + remaining = args.instance_count - index - 1 + _await_resources( + args.node_container, + capacity["CPU"] - args.cpu * remaining, + capacity["Memory"] - args.memory * remaining, + ) + if _checkpoint_views( + args.node_container, args.host_checkpoint_root, sandbox.id + )["container"]: + raise AssertionError(f"interleaved sandbox {index} left checkpoint.img after delete") + + return { + "instances": [ + {key: value for key, value in item.items() if key != "sandbox"} for item in metadata + ], + "actions": action_evidence, + } + + +def _resource_view(node_container: str) -> dict[str, Any]: + address = base._node_ip(node_container) + response = base._docker_exec( + node_container, + "curl", + "-fsS", + f"http://{address}:22770/global-scheduler/resources", + ) + payload = json.loads(response.stdout) + resource = payload["resource"] + + def values(section: str) -> dict[str, int]: + entries = resource[section]["resources"] + return { + name: int(entries.get(name, {}).get("scalar", {}).get("value", 0)) + for name in ("CPU", "Memory", "storage") + } + + return { + "requestID": payload.get("requestID"), + "revision": int(resource.get("revision", 0)), + "capacity": values("capacity"), + "allocatable": values("allocatable"), + "actualUse": values("actualUse"), + } + + +def _assert_allocatable(view: dict[str, Any], cpu: int, memory: int) -> None: + actual = view["allocatable"] + if actual["CPU"] != cpu or actual["Memory"] != memory: + raise AssertionError( + f"resource view mismatch: expected CPU={cpu} Memory={memory}, got {view!r}" + ) + + +def _await_resources( + node_container: str, cpu: int, memory: int, timeout: float = 5.0 +) -> tuple[dict[str, Any], float]: + started = time.perf_counter() + last = _resource_view(node_container) + while last["allocatable"]["CPU"] != cpu or last["allocatable"]["Memory"] != memory: + if time.perf_counter() - started >= timeout: + _assert_allocatable(last, cpu, memory) + time.sleep(0.05) + last = _resource_view(node_container) + return last, round(time.perf_counter() - started, 6) + + +def _container_checkpoint_images( + node_container: str, instance_id: str = "" +) -> list[dict[str, Any]]: + result = base._docker_exec( + node_container, + "find", + "/home/akernel/sandboxd/root/checkpoints", + "-type", + "f", + "-name", + "checkpoint.img", + "-printf", + "%p\t%s\n", + check=False, + ) + if result.returncode not in (0, 1): + raise RuntimeError(f"checkpoint cache scan failed: {result.stderr}") + images: list[dict[str, Any]] = [] + for line in result.stdout.splitlines(): + path, size = line.rsplit("\t", 1) + if instance_id and instance_id not in path: + continue + relative = path.removeprefix("/home/akernel/sandboxd/root/checkpoints/") + images.append({"path": path, "relativePath": relative, "size": int(size)}) + return sorted(images, key=lambda item: item["relativePath"]) + + +def _host_checkpoint_images( + host_checkpoint_root: Path, instance_id: str = "" +) -> list[dict[str, Any]]: + if not host_checkpoint_root.exists(): + return [] + images: list[dict[str, Any]] = [] + for path in host_checkpoint_root.rglob("checkpoint.img"): + relative = path.relative_to(host_checkpoint_root).as_posix() + if instance_id and instance_id not in relative: + continue + images.append( + { + "path": str(path), + "relativePath": relative, + "size": path.stat().st_size, + } + ) + return sorted(images, key=lambda item: item["relativePath"]) + + +def _assert_checkpoint_views_agree(views: dict[str, list[dict[str, Any]]]) -> None: + def facts(name: str) -> set[tuple[str, int]]: + return { + (str(item["relativePath"]), int(item["size"])) + for item in views[name] + } + + container_facts = facts("container") + host_facts = facts("host") + if container_facts != host_facts: + raise AssertionError( + "checkpoint bind-mount views disagree: " + f"container={sorted(container_facts)!r} host={sorted(host_facts)!r}" + ) + + +def _checkpoint_views( + node_container: str, host_checkpoint_root: Path, instance_id: str = "" +) -> dict[str, list[dict[str, Any]]]: + views = { + "container": _container_checkpoint_images(node_container, instance_id), + "host": _host_checkpoint_images(host_checkpoint_root, instance_id), + } + _assert_checkpoint_views_agree(views) + return views + + +def _percentile(values: list[float], fraction: float) -> float: + ordered = sorted(values) + index = max(0, min(len(ordered) - 1, int(round((len(ordered) - 1) * fraction)))) + return ordered[index] + + +def _summary(values: list[float]) -> dict[str, float]: + return { + "count": len(values), + "min": min(values), + "median": statistics.median(values), + "p95": _percentile(values, 0.95), + "max": max(values), + "mean": statistics.fmean(values), + } + + +def _performance_summary(single: dict[str, Any], interleaved: dict[str, Any]) -> dict[str, Any]: + cycles = single["cycles"] + actions = interleaved["actions"] + return { + "singleLoop": { + name: _summary([float(cycle[name]) for cycle in cycles]) + for name in ( + "pauseSeconds", + "resumeSeconds", + "firstFileSeconds", + "firstExecSeconds", + "firstPublicSeconds", + ) + }, + "interleavedPause": _summary( + [float(action["pauseSeconds"]) for action in actions if "pauseSeconds" in action] + ), + "interleavedResume": _summary( + [float(action["resumeSeconds"]) for action in actions if "resumeSeconds" in action] + ), + } + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/standalone/start.sh b/deploy/standalone/start.sh index f4cc29f..e592dde 100755 --- a/deploy/standalone/start.sh +++ b/deploy/standalone/start.sh @@ -296,6 +296,7 @@ start_node_container() { --net bridge \ --restart always \ -e AKS_LOCAL_MODE="true" \ + -e AKERNEL_ENABLE_PAUSE_RESUME="${AKERNEL_ENABLE_PAUSE_RESUME:-false}" \ -e TRAEFIK_MODE="http" \ -e TRAEFIK_HTTP_ENTRYPOINT="web" \ -e TRAEFIK_ENABLE_TLS="false" \ @@ -358,7 +359,7 @@ http: akernel-frontend: entryPoints: - websecure - rule: "PathPrefix(\`/terminal\`) || PathPrefix(\`/api/instances\`) || PathPrefix(\`/api/jobs\`) || PathPrefix(\`/functions\`) || PathPrefix(\`/api-docs\`) || PathPrefix(\`/admin/v1/functions\`) || PathPrefix(\`/serverless/v1/functions\`) || PathPrefix(\`/serverless/v1/stream\`) || PathPrefix(\`/serverless/v1/componentshealth\`) || PathPrefix(\`/serverless/v1/posix\`) || PathPrefix(\`/serverless/v2\`) || PathPrefix(\`/frontend/v1/instance\`) || PathPrefix(\`/datasystem/v1\`) || PathPrefix(\`/app/v1\`) || PathPrefix(\`/client/v1/lease\`) || PathPrefix(\`/invocations\`) || PathPrefix(\`/global-scheduler\`) || Path(\`/healthz\`)" + rule: "PathPrefix(\`/terminal\`) || PathPrefix(\`/api/instances\`) || PathPrefix(\`/api/jobs\`) || PathPrefix(\`/api/sandbox/v1\`) || PathPrefix(\`/direct\`) || PathPrefix(\`/functions\`) || PathPrefix(\`/api-docs\`) || PathPrefix(\`/admin/v1/functions\`) || PathPrefix(\`/serverless/v1/functions\`) || PathPrefix(\`/serverless/v1/stream\`) || PathPrefix(\`/serverless/v1/componentshealth\`) || PathPrefix(\`/serverless/v1/posix\`) || PathPrefix(\`/serverless/v2\`) || PathPrefix(\`/frontend/v1/instance\`) || PathPrefix(\`/datasystem/v1\`) || PathPrefix(\`/app/v1\`) || PathPrefix(\`/client/v1/lease\`) || PathPrefix(\`/invocations\`) || PathPrefix(\`/global-scheduler\`) || Path(\`/healthz\`)" service: akernel-frontend tls: {} sandbox-router: diff --git a/deploy/standalone/tests/test-pause-resume-wiring.sh b/deploy/standalone/tests/test-pause-resume-wiring.sh new file mode 100644 index 0000000..be67589 --- /dev/null +++ b/deploy/standalone/tests/test-pause-resume-wiring.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Copyright (c) 2026 Ant Group Corporation. +# +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +bootstrap="${repo_root}/builder/scripts/yr_node_bootstrap.sh" +pause_resume_helper="${repo_root}/builder/scripts/yr_pause_resume_args.sh" +service="${repo_root}/builder/systemd_services/yuanrong.service" +start="${repo_root}/deploy/standalone/start.sh" +tmp_dir="$(mktemp -d)" +trap 'rm -rf "${tmp_dir}"' EXIT + +grep -Fq 'AKERNEL_ENABLE_PAUSE_RESUME:-false' "${bootstrap}" +grep -Fq '/home/yuanrong/.akernel-rrt-capable' "${bootstrap}" +grep -Fq '/home/akernel/sandboxd/root/checkpoints' "${bootstrap}" +grep -Fq 'source /root/yr_pause_resume_args.sh' "${bootstrap}" +grep -Fq 'configure_pause_resume_args' "${bootstrap}" + +source "${pause_resume_helper}" +capability_file="${tmp_dir}/rrt-capable" +checkpoint_dir="${tmp_dir}/checkpoints" +touch "${capability_file}" +configure_pause_resume_args true "${capability_file}" "${checkpoint_dir}" true +expected="--enable_sandbox_pause_resume true --snapshot_storage_backend datasystem --checkpoint_dir ${checkpoint_dir} --data_system_enable true" +if [[ "${standalone_pause_resume_args[*]}" != "${expected}" ]]; then + echo "standalone does not default to DataSystem snapshot storage" >&2 + exit 1 +fi +grep -Eq '^PassEnvironment=.*AKERNEL_ENABLE_PAUSE_RESUME' "${service}" +grep -Fq 'AKERNEL_ENABLE_PAUSE_RESUME="${AKERNEL_ENABLE_PAUSE_RESUME:-false}"' "${start}" +grep -Fq 'PathPrefix(\`/api/sandbox/v1\`)' "${start}" +grep -Fq 'PathPrefix(\`/direct\`)' "${start}" + +echo "pause/resume standalone wiring contract passed" diff --git a/deploy/standalone/tests/test_pause_resume_memory_state.py b/deploy/standalone/tests/test_pause_resume_memory_state.py new file mode 100644 index 0000000..3b79c60 --- /dev/null +++ b/deploy/standalone/tests/test_pause_resume_memory_state.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +import sys + + +MODULE_PATH = Path(__file__).parents[1] / "pause_resume_function_matrix_e2e.py" +sys.path.insert(0, str(MODULE_PATH.parent)) +SPEC = importlib.util.spec_from_file_location("pause_resume_function_matrix_e2e", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def test_parse_memory_state() -> None: + state = MODULE._parse_memory_state("2\t314\t42\t33554432\t123456\t90\n") + + assert state == { + "sequence": 2, + "pid": 314, + "counter": 42, + "length": 33_554_432, + "checksum": 123_456, + "sentinelByte": 90, + } + + +def test_assert_memory_state_continuity_accepts_same_process_and_memory() -> None: + before = { + "sequence": 1, + "pid": 314, + "counter": 42, + "length": 33_554_432, + "checksum": 123_456, + "sentinelByte": 90, + } + after = {**before, "sequence": 2} + + MODULE._assert_memory_state_continuity(before, after) + + +def test_assert_memory_state_continuity_rejects_reinitialized_process() -> None: + before = { + "sequence": 1, + "pid": 314, + "counter": 42, + "length": 33_554_432, + "checksum": 123_456, + "sentinelByte": 90, + } + after = {**before, "sequence": 2, "pid": 315, "counter": 41} + + try: + MODULE._assert_memory_state_continuity(before, after) + except AssertionError as exc: + assert "memory process identity/state changed" in str(exc) + else: + raise AssertionError("reinitialized process was accepted") diff --git a/docs/superpowers/plans/2026-08-12-standalone-rrt-pause-resume-e2e.md b/docs/superpowers/plans/2026-08-12-standalone-rrt-pause-resume-e2e.md new file mode 100644 index 0000000..ba2afd8 --- /dev/null +++ b/docs/superpowers/plans/2026-08-12-standalone-rrt-pause-resume-e2e.md @@ -0,0 +1,126 @@ +# Standalone RRT Pause/Resume E2E Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build an amd64 AKernel all-in-one image from Buildkite #215 artifacts and prove two complete Sandbox SDK pause/resume cycles on a native Linux x86_64 standalone host. + +**Architecture:** Extend the existing two-stage Docker build with a checksum-pinned RRT wheel and an RRT-specific services file. Apply a checksum/context-gated compatibility patch to the #215 process deployment scripts, opt the standalone service into DataSystem-backed pause/resume, compile the verified sandboxd checkpoint revision, and run a host-side SDK continuity test against the standalone gateway. + +**Tech Stack:** Bash, Docker/BuildKit, Dockerfile, EROFS, systemd, openYuanRong C++ FunctionSystem, Rust RRT, sandboxd Go RPC, runsc, DataSystem, Python `yr_sandbox`. + +## Global Constraints + +- Target only native Linux x86_64; Darwin arm64 and QEMU are not E2E evidence. +- Use Buildkite #215 core SHA `39ba1cf8323ac4e784117867ecd806ec392da05aa2fd87130f8830eb56310895` and RRT SHA `51e16a48a98ff89497e268e939ae046205c9ba287e8b0b571eec3048e6d38ae7`. +- Keep the C++ FunctionSystem; RRT is the sandbox runtime, not Rust FunctionSystem. +- Keep RRT capability conditional at build time and pause/resume disabled by default at runtime. +- Use DataSystem and checkpoint root `/home/akernel/sandboxd/root/checkpoints`. +- Preserve unrelated `src/yuanrong` dirt and the untracked top-level `sandboxd/` worktree. +- Never log or commit cloud credentials, IAM seed, SDK token, registry auth, or private keys. + +--- + +### Task 1: RRT Build Inputs and Runtime Rootfs + +**Files:** +- Modify: `Makefile` +- Modify: `deploy/scripts/build-image.sh` +- Modify: `builder/runtime.Dockerfile` +- Create: `builder/config/yr_services_rrt.yaml` +- Modify: `builder/node.Dockerfile` +- Create: `deploy/scripts/tests/test-build-image-rrt.sh` + +**Interfaces:** +- Consumes: `OPEN_YR_RRT_WHEEL_URL` and `OPEN_YR_RRT_WHEEL_SHA256` as an all-or-nothing pair. +- Produces: `/opt/openyuanrong-rrt/rrt-runtime` in `yr-runtime-rootfs.img` and an image marker `/home/yuanrong/.akernel-rrt-capable`. + +- [ ] Write a shell test that substitutes a fake `docker`, captures both build argv arrays, and asserts pair validation, runtime build args, node capability arg, and default-build compatibility. +- [ ] Run `bash deploy/scripts/tests/test-build-image-rrt.sh`; expect failure because RRT CLI inputs are unknown. +- [ ] Add Make variables/flags, paired validation, runtime Docker build args, checksum-pinned wheel extraction, amd64 ELF validation, RRT services selection, and capability marker. +- [ ] Run the shell test and `bash -n deploy/scripts/build-image.sh`; expect success. +- [ ] Commit only Task 1 files with a signed Conventional Commit. + +### Task 2: #215 Process-Mode Compatibility Wiring + +**Files:** +- Create: `builder/patches/openyuanrong-core-6dfa49681774-pause-resume-process.patch` +- Create: `builder/scripts/apply-openyuanrong-pause-resume-patch.sh` +- Modify: `builder/node.Dockerfile` +- Create: `builder/scripts/tests/test-openyuanrong-pause-resume-patch.sh` + +**Interfaces:** +- Consumes: extracted #215 `yr/deploy/process/config.sh` and `yr/functionsystem/deploy/install.sh`. +- Produces: process options `enable_sandbox_pause_resume`, `snapshot_storage_backend`, and `checkpoint_dir`, forwarded into the merged FunctionProxy/FunctionAgent/RuntimeManager composition. + +- [ ] Write a fixture test that downloads or reads the checksum-pinned core wheel, applies the patch to a temporary extracted `yr/`, and asserts each option is parsed/exported exactly once and present in the merged process argv. +- [ ] Run the test; expect failure because patch/apply helper do not exist. +- [ ] Add the exact-context patch and helper; require the expected core SHA and fail if patch dry-run or postconditions fail. +- [ ] Invoke the helper in `builder/node.Dockerfile` only for RRT-capable builds. +- [ ] Run the fixture test and `git diff --check`; expect success. +- [ ] Commit only Task 2 files with a signed Conventional Commit. + +### Task 3: Checkpoint-Capable sandboxd Source + +**Files:** +- Modify gitlink: `src/sandboxd` + +**Interfaces:** +- Consumes: verified sandboxd commit `29c7b219` from branch `codex/native-gvisor-checkpoint-v1`. +- Produces: wire-compatible Checkpoint/Restore/List RPC binaries compiled by `builder/node.Dockerfile`. + +- [ ] Fetch the verified fork branch inside `src/sandboxd` and inspect the target commit/tests without copying from the untracked sibling checkout. +- [ ] Run focused Go tests for protobuf compatibility, checkpoint artifact/state, Checkpoint/Restore RPC, and runsc handler. +- [ ] Checkout the verified target commit detached in the submodule and verify `git diff --submodule=log` changes only the gitlink. +- [ ] Commit the gitlink with a signed Conventional Commit. + +### Task 4: Standalone Feature Gate and E2E Runner + +**Files:** +- Modify: `builder/scripts/yr_node_bootstrap.sh` +- Modify: `builder/systemd_services/yuanrong.service` +- Modify: `deploy/standalone/start.sh` +- Modify: `deploy/standalone/README.md` +- Create: `deploy/standalone/pause_resume_e2e.py` +- Create: `deploy/standalone/tests/test_pause_resume_wiring.sh` + +**Interfaces:** +- Consumes: `AKERNEL_ENABLE_PAUSE_RESUME=true` and image capability marker. +- Produces: gate/backend/root args to `yr start` plus a JSON E2E report with two snapshots and continuity assertions. + +- [ ] Write a shell wiring test that sources/extracts command construction and asserts default-off, explicit-on args, systemd environment pass-through, and early rejection without the capability marker. +- [ ] Run the shell test; expect failure because the environment and args are absent. +- [ ] Refactor bootstrap argv into an array, append the three pause/resume options only when enabled, pass the variable through systemd and standalone, and document usage. +- [ ] Implement the Python runner using `yr_sandbox.Sandbox(runtime="runsc")`: marker file, stdin-blocked background PID, Pause, watcher PAUSED, Resume, PID continuation, marker/new-command checks, second cycle, Delete, and credential-free JSON. +- [ ] Run wiring tests, `bash -n`, and Python compile checks; expect success. +- [ ] Commit Task 4 files with a signed Conventional Commit. + +### Task 5: Native x86 Image Build and Static Verification + +**Files:** +- Runtime evidence only under remote `/root/akernel-e2e/`; do not add credentials or generated output to Git. + +**Interfaces:** +- Consumes: repository commit, #215 core/RRT URLs and SHA values. +- Produces: local remote image `akernel-all-in-one:pause-resume-215`. + +- [ ] Sync a clean source snapshot plus initialized submodules to `47.110.151.176` without `.git` secrets, local deployment state, or untracked sibling worktrees. +- [ ] Run the complete local script/unit gates on the x86 host. +- [ ] Run `make build` with the exact core/RRT URL/SHA pair and capture a non-secret build log. +- [ ] Inspect the image for amd64 architecture, OCI revisions, RRT marker/binary, one RRT service slot, patched process options, and sandboxd Checkpoint RPC strings/descriptor. +- [ ] Record image ID and size in a temporary evidence directory. + +### Task 6: Standalone Pause/Resume End-to-End + +**Files:** +- Runtime evidence only under remote `/root/akernel-e2e/evidence/`. + +**Interfaces:** +- Consumes: `akernel-all-in-one:pause-resume-215`, checksum-pinned `openyuanrong-sandbox` wheel. +- Produces: passed JSON report and redacted diagnostic bundle. + +- [ ] Start standalone with `AKERNEL_ENABLE_PAUSE_RESUME=true` and the local image; verify systemd, Frontend, DataSystem, sandboxd, and gateway readiness. +- [ ] Inspect actual FunctionProxy argv for gate `true`, backend `datasystem`, and the exact shared checkpoint root. +- [ ] Create a checksum-verified host venv for the #215 Sandbox SDK and run `pause_resume_e2e.py` against the Traefik bridge IP/token. +- [ ] Require two distinct snapshots, authoritative PAUSED/RUNNING transitions, pre-pause PID continuation, marker persistence, new command health, and successful Delete. +- [ ] Collect redacted YuanRong/sandboxd/runsc/DataSystem logs and checkpoint directory metadata keyed by sandbox/snapshot ID. +- [ ] Stop standalone, verify no leaked containers/mounts, and report exact pass/fail evidence before the ECS reclaim deadline. diff --git a/docs/superpowers/plans/2026-08-14-akernel-buildkite-image-pipeline.md b/docs/superpowers/plans/2026-08-14-akernel-buildkite-image-pipeline.md new file mode 100644 index 0000000..71902ee --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-akernel-buildkite-image-pipeline.md @@ -0,0 +1,269 @@ +# AKernel Buildkite Image Pipeline Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a repository-owned Buildkite pipeline that consumes YuanRong release or Buildkite artifacts, pushes one universal AKernel image, and publishes sandbox SDK and deployment bundles. + +**Architecture:** A source resolver normalizes both YuanRong sources into one JSON manifest. A privileged image job consumes that manifest and publishes a digest-confirmed SWR image; a separate deterministic packager creates the requested standalone and Helm bundles from the two manifests. + +**Tech Stack:** Buildkite YAML and agent CLI, Bash, Python 3 standard library, Docker, SWR, GitHub Releases, PyPI JSON, Buildkite REST API. + +## Global Constraints + +- Build only one universal image for standalone and Kubernetes deployment. +- Default to `swr.cn-southwest-2.myhuaweicloud.com/openyuanrong/akernel-all-in-one`. +- Use the RRT runtime profile; do not build the Python runtime profile. +- Keep credentials out of Git, Buildkite metadata, command output, and artifacts. +- Resolve Buildkite artifacts from a passed build's metadata rather than scraping logs. +- Use the existing C++ FunctionSystem YuanRong package path; do not trigger a new Rust FunctionSystem build. +- Work on `codex/yuanrong-downloaders` in the existing isolated worktree. + +--- + +### Task 1: Normalize YuanRong Sources + +**Files:** +- Create: `.buildkite/scripts/resolve_yuanrong.py` +- Create: `.buildkite/tests/test_resolve_yuanrong.py` + +**Interfaces:** +- Consumes: release version or Buildkite organization, pipeline, build number, and API token. +- Produces: `artifact-manifest.json` and a downloaded `openyuanrong_sandbox-*.whl` under an output directory. + +- [ ] **Step 1: Write failing resolver behavior tests** + + Cover a Release fixture with checksum files and PyPI JSON, a passed Buildkite + fixture with all three metadata keys, rejection of a non-passed build, and + rejection of missing/duplicate artifact entries. Use a real local HTTP + server and literal expected manifest fields. + +- [ ] **Step 2: Run the resolver tests and verify RED** + + Run: `python3 -m unittest .buildkite/tests/test_resolve_yuanrong.py -v` + + Expected: failure because `.buildkite/scripts/resolve_yuanrong.py` does not + exist. + +- [ ] **Step 3: Implement the minimal resolver** + + Implement explicit CLI subcommands `release` and `buildkite`; URL manifest + parsing; streamed downloads; SHA-256 validation; PyPI wheel selection; and + stable JSON serialization. Redact authorization values from all errors. + +- [ ] **Step 4: Run the resolver tests and verify GREEN** + + Run: `python3 -m unittest .buildkite/tests/test_resolve_yuanrong.py -v` + + Expected: all resolver cases pass. + +### Task 2: Plumb Arbitrary Release Inputs into the Image Build + +**Files:** +- Modify: `Makefile` +- Modify: `deploy/scripts/build-image.sh` +- Modify: `deploy/scripts/tests/test-build-image-rrt.sh` + +**Interfaces:** +- Consumes: `OPEN_YR_VERSION`, `RRT_RUNTIME_URL`, and + `RRT_RUNTIME_SHA256` Make variables. +- Produces: matching Docker build arguments for `builder/runtime.Dockerfile` + and `builder/node.Dockerfile`. + +- [ ] **Step 1: Add failing observable argument-propagation tests** + + Extend the build contract test to run the Make/build wrapper with a fake + Docker executable and assert that literal release version, runtime URL, and + checksum values reach the runtime and node Docker invocations. + +- [ ] **Step 2: Run the build test and verify RED** + + Run: `bash deploy/scripts/tests/test-build-image-rrt.sh` + + Expected: failure because the new inputs are not accepted or forwarded. + +- [ ] **Step 3: Implement minimal Make and shell plumbing** + + Add paired CLI arguments and enforce URL/checksum pairing. Pass the version + to both Docker builds and raw RRT inputs only to the runtime build. + +- [ ] **Step 4: Run the build test and verify GREEN** + + Run: `bash deploy/scripts/tests/test-build-image-rrt.sh` + + Expected: pass with both existing wheel overrides and new release inputs. + +### Task 3: Build and Push the Universal Image + +**Files:** +- Create: `.buildkite/scripts/docker_job_helpers.sh` +- Create: `.buildkite/scripts/build_and_push.sh` +- Create: `.buildkite/tests/test_build_and_push.py` + +**Interfaces:** +- Consumes: normalized artifact manifest, image repository/tag inputs, SWR + credential environment, and the AKernel checkout. +- Produces: `artifacts/image/image-manifest.json` containing tag, digest, + AKernel commit, YuanRong source, and component configuration. + +- [ ] **Step 1: Write failing image-wrapper tests** + + Run the wrapper with fake `git`, `make`, and `docker` executables. Assert the + exact Make variables for both `rrt.kind=runtime` and `rrt.kind=wheel`, the + generated safe tag, login-before-push ordering, and digest-confirmed image + manifest. Add rejection cases for malformed tags and missing credentials. + +- [ ] **Step 2: Run the image-wrapper tests and verify RED** + + Run: `python3 -m unittest .buildkite/tests/test_build_and_push.py -v` + + Expected: failure because the wrapper is missing. + +- [ ] **Step 3: Implement Docker lifecycle and image publication** + + Start dockerd only when needed, initialize `src/sandboxd` and + `src/distill-fs`, invoke `make build` with normalized inputs, authenticate + with password stdin or injected Docker config, push, inspect the registry + digest, and write the manifest atomically. + +- [ ] **Step 4: Run the image-wrapper tests and verify GREEN** + + Run: `python3 -m unittest .buildkite/tests/test_build_and_push.py -v` + + Expected: all wrapper cases pass. + +### Task 4: Package Standalone and Helm Deployments + +**Files:** +- Create: `.buildkite/scripts/package_deployments.py` +- Create: `.buildkite/tests/test_package_deployments.py` + +**Interfaces:** +- Consumes: checkout, artifact manifest, image manifest, sandbox SDK wheel, + and normalized target list. +- Produces: standalone tarball, Helm tgz, copied manifests, and SHA256SUMS. + +- [ ] **Step 1: Write failing archive behavior tests** + + Generate small real fixture trees and assert exact archive member names, + image tag values, digest records, target filtering, SDK inclusion, and + checksums. Include rejection of a sandbox wheel whose digest differs from + the artifact manifest. + +- [ ] **Step 2: Run packaging tests and verify RED** + + Run: `python3 -m unittest .buildkite/tests/test_package_deployments.py -v` + + Expected: failure because the packager is missing. + +- [ ] **Step 3: Implement deterministic packaging** + + Normalize target names, validate both manifests, verify the SDK wheel, + generate `image.env` and `values.image.yaml`, create archives with stable + relative roots, and calculate SHA256SUMS over final deliverables. + +- [ ] **Step 4: Run packaging tests and verify GREEN** + + Run: `python3 -m unittest .buildkite/tests/test_package_deployments.py -v` + + Expected: all archive cases pass. + +### Task 5: Generate and Document the Buildkite Pipeline + +**Files:** +- Create: `.buildkite/pipeline.yml` +- Create: `.buildkite/pipeline.sh` +- Create: `.buildkite/tests/test_pipeline.py` +- Create: `.buildkite/README.md` +- Modify: `README.md` +- Modify: `AGENTS.md` +- Modify: `Makefile` + +**Interfaces:** +- Consumes: documented Buildkite environment variables and Kubernetes + secrets `swr-credentials` / `swr-pull-secret`. +- Produces: valid dynamic Buildkite YAML with resolve, image, and packaging + steps plus a local `make buildkite-check` gate. + +- [ ] **Step 1: Write failing dynamic-pipeline tests** + + Execute the generator for Release and Buildkite inputs, parse stdout with + PyYAML, and assert dependency ordering, target forwarding, bounded + privileged image resources, secret references, artifact upload commands, + and early rejection of invalid source/target/build-number values. + +- [ ] **Step 2: Run pipeline tests and verify RED** + + Run: `python3 -m unittest .buildkite/tests/test_pipeline.py -v` + + Expected: failure because the pipeline generator is missing. + +- [ ] **Step 3: Implement bootstrap, dynamic YAML, docs, and quality gate** + + Keep the checked-in bootstrap stable, emit the three-step DAG, document all + inputs and artifact contracts, add the CI directory to project layout + guidance, and add `make buildkite-check` to run Python tests plus shell + syntax validation. + +- [ ] **Step 4: Run the local Buildkite gate and verify GREEN** + + Run: `make buildkite-check` + + Expected: all Buildkite tests and shell syntax checks pass. + +### Task 6: Verify, Commit, Push, and Exercise Buildkite + +**Files:** +- Modify only files listed in Tasks 1-5. + +**Interfaces:** +- Consumes: the completed local branch and Buildkite API credentials. +- Produces: a pushed feature branch, a new `akernel-image` Buildkite pipeline, + two accepted builds, pushed image references, and downloadable artifacts. + +- [ ] **Step 1: Run the full local verification set** + + Run: + + ```bash + make buildkite-check + bash builder/downloaders/tests/test-openyuanrong-downloaders.sh + bash deploy/scripts/tests/test-build-image-rrt.sh + make deploy-script-check + git diff --check + ``` + + Expected: every command exits zero. + +- [ ] **Step 2: Commit with project-required DCO sign-off** + + Stage only feature files, generate a Conventional Commit message with prose + body, and commit using `git commit -s`. + +- [ ] **Step 3: Push the feature branch** + + Push `codex/yuanrong-downloaders` to `origin` without force. + +- [ ] **Step 4: Create the Buildkite pipeline** + + Create `openyuanrong/akernel-image` in the existing default cluster, point + it to `https://github.com/inclusionAI/AKernel.git`, use the checked-in + bootstrap configuration, and set the initial default branch to + `codex/yuanrong-downloaders`. + +- [ ] **Step 5: Run Release acceptance** + + Trigger with `YR_SOURCE=release`, `YR_VERSION=0.9.7`, and both deployment + targets. Verify the build contains resolve, image, and packaging jobs; the + remote image digest exists; and all five artifact classes are present. + +- [ ] **Step 6: Run Buildkite-source acceptance** + + Trigger with `YR_SOURCE=buildkite`, `YR_PIPELINE=yuanrong-jcl`, and a passed + YuanRong build number. Verify the normalized manifest names that exact + source build, the sandbox SDK matches its OBS metadata, the image is pushed, + and both deployment bundles are downloadable. + +- [ ] **Step 7: Collect final evidence** + + Save build URLs, job states, artifact names, image tags/digests, and safe + manifest summaries locally without copying credentials. diff --git a/docs/superpowers/plans/2026-08-14-buildkite-large-dependency-pvc-cache.md b/docs/superpowers/plans/2026-08-14-buildkite-large-dependency-pvc-cache.md new file mode 100644 index 0000000..f6e1ecf --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-buildkite-large-dependency-pvc-cache.md @@ -0,0 +1,386 @@ +# Buildkite Large Dependency PVC Cache Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Cache the checksum-pinned Kata Containers 4.0.0 amd64 static archive, +gVisor `runsc`, and OpenTelemetry Collector contrib archive on a persistent +Buildkite Kubernetes volume and consume them through a read-only BuildKit named +context. + +**Architecture:** A host-side downloader owns SHA-256/SHA-512 validation and +atomic cache publication. The Buildkite image job mounts one named PVC, passes +it to the existing build driver, and BuildKit exposes it read-only to the Kata, +gVisor, and OpenTelemetry stages; each Docker consumer verifies its pinned +digest again before installation. `/var/lib/docker` remains a per-job +`emptyDir`, and builds without a configured cache retain the upstream-download +fallback. + +**Tech Stack:** Bash, Docker BuildKit named contexts, Dockerfile bind mounts, Buildkite Kubernetes PodSpec patches, Kubernetes PVC, Python `unittest`, shell contract tests. + +## Global Constraints + +- The repository-pinned URL and digest remain authoritative; the PVC is only a performance cache. +- Cache `kata-static-4.0.0-amd64.tar.zst` (`1,952,994,060` bytes, SHA-256 `2c3b9dfeba355582b40aee462b12916c9740654d0230f696adf719d67b063a8c`) at `kata/4.0.0/amd64/2c3b9dfeba355582b40aee462b12916c9740654d0230f696adf719d67b063a8c/kata-static-4.0.0-amd64.tar.zst`. +- Cache gVisor `runsc` `release-20260706.0` x86_64 (`130,918,823` bytes, SHA-512 `73938c145ebe554cf61a01da455688f4b732eebdf7b1b635bdef5b195868b363d8cb400e3d92ed1f377b78996805556c247a4849583910cb04e92b156053033e`) at `gvisor/release-20260706.0/x86_64/73938c145ebe554cf61a01da455688f4b732eebdf7b1b635bdef5b195868b363d8cb400e3d92ed1f377b78996805556c247a4849583910cb04e92b156053033e/runsc`. +- Cache `otelcol-contrib_0.120.0_linux_amd64.tar.gz` (`80,901,637` bytes, SHA-256 `81bf885bc9a86705feb3c113c5a356571390e3601eb651ffcf2b3428f6571adb`) at `otelcol-contrib/0.120.0/linux-amd64/81bf885bc9a86705feb3c113c5a356571390e3601eb651ffcf2b3428f6571adb/otelcol-contrib_0.120.0_linux_amd64.tar.gz`. +- Cache identity includes component, version, architecture, digest, and filename; the three entries total `2,164,814,520` bytes, leaving `8,572,603,720` bytes on the 10 GiB PV before filesystem overhead. +- Only a checksum-verified temporary file may be atomically renamed to the final cache path. +- Mount the PVC read-write only in the image command container; checkout, YuanRong resolution, and packaging must not mount it. +- Mount the cache read-only in the Docker stage and verify SHA-256 or SHA-512, as pinned for the artifact, again. +- Keep `/var/lib/docker` on a per-job `emptyDir`; never share a Docker data root between jobs. +- `AKERNEL_DEPENDENCY_CACHE_DIR` unset means no host prefetch and preserves the upstream fallback. +- `GVISOR_RELEASE` and `GVISOR_AMD64_SHA512`, and separately `OTELCOL_CONTRIB_VERSION` and `OTELCOL_CONTRIB_SHA256`, must be overridden as pairs; a version-only or digest-only override is rejected before Docker starts. +- A corrupt existing cache entry is not trusted: the host verifies it, then downloads, verifies, and atomically replaces it on a miss. +- Keep WireGuard, `NO_PROXY`, exact submodule checkout, C++ YuanRong default, Kata, and NVIDIA defaults unchanged. +- Never read, print, or regenerate `AKERNEL_WG_CONFIG`. +- Do not stop or release the Hong Kong egress ECS. + +## Dependency audit decision + +- Implement now: Kata Containers 4.0.0 amd64 static archive, 1,952,994,060 bytes, SHA-256 pinned. +- Implement now: gVisor `runsc` `release-20260706.0` x86_64, 130,918,823 bytes, SHA-512 pinned; retain its existing binary packaging and `/usr/local/bin/runsc` installation. +- Implement now: OpenTelemetry Collector contrib 0.120.0 linux amd64 archive, 80,901,637 bytes, SHA-256 pinned; retain its existing configuration and systemd wiring. +- The three immutable archives use about 2.165 GB decimal, so the existing 10 GiB static local PV remains sufficient. +- Next candidate, outside this plan: openYuanRong core 0.9.7 x86_64 wheel, official non-prerelease, 234 MB, already checksum-pinned. +- Defer managed CPython: five optional Python-profile assets total about 163 MB, are not built by RRT, and several patch versions need maintenance review. +- Exclude Docker base images: mirror them to Guiyang SWR or add a safe BuildKit layer cache rather than treating registry layers as archive-cache files. + +## File structure + +- Create `builder/downloaders/cache-verified-download.sh`: generic verified atomic download primitive. +- Create `builder/downloaders/tests/test-cache-verified-download.sh`: isolated behavior tests with fake `curl`. +- Modify `deploy/scripts/build-image.sh`: derive the Kata, gVisor, and + OpenTelemetry cache paths, prefetch them, and pass a named context. +- Modify `builder/node.Dockerfile`: read a cached artifact or use the existing + URL fallback for Kata, gVisor, and OpenTelemetry. +- Modify `deploy/scripts/tests/test-build-image-rrt.sh`: assert all three + cache paths, build arguments, read-only mounts, and Dockerfile contracts. +- Modify `.buildkite/pipeline.sh`: mount the named PVC only in the image command container. +- Modify `.buildkite/tests/test_pipeline.py`: prove PVC isolation and environment wiring. +- Create `.buildkite/kubernetes/akernel-dependency-cache-pvc.yaml`: 10 GiB `ReadWriteOnce` claim. +- Modify `.buildkite/README.md`: document prerequisites and cache semantics. + +--- + +### Task 1: Checksum-verified atomic cache primitive + +**Files:** +- Create: `builder/downloaders/tests/test-cache-verified-download.sh` +- Create: `builder/downloaders/cache-verified-download.sh` + +**Interfaces:** +- Consumes: `cache-verified-download.sh URL SHA256 DESTINATION` +- Produces: `cache-hit ` or `cache-fill ` and a final file only after verification. + +- [ ] **Step 1: Write the failing downloader test** + +Create a fake `curl` that copies `FAKE_CURL_SOURCE` to the path following `-o` and increments `FAKE_CURL_COUNT`. Verify fill, hit without another download, corrupt-hit replacement, checksum mismatch, interrupted-download cleanup, and two concurrent cache misses publishing identical valid content without leftover `*.part.*` files. The central assertions are: + +```bash +expected_sha="$(sha256sum "${fixture}" | awk '{print $1}')" +"${DOWNLOADER}" "https://example.invalid/archive" "${expected_sha}" "${cache_file}" +[[ "$(cat "${count_file}")" == 1 ]] +"${DOWNLOADER}" "https://example.invalid/archive" "${expected_sha}" "${cache_file}" +[[ "$(cat "${count_file}")" == 1 ]] +printf 'corrupt\n' >"${cache_file}" +"${DOWNLOADER}" "https://example.invalid/archive" "${expected_sha}" "${cache_file}" +[[ "$(cat "${count_file}")" == 2 ]] +``` + +- [ ] **Step 2: Run the test and record RED** + +Run: `bash builder/downloaders/tests/test-cache-verified-download.sh` + +Expected: FAIL because the downloader does not exist. + +- [ ] **Step 3: Implement the downloader** + +Implement the following strict flow, including URL, digest, destination, and temporary-file validation: + +```bash +if [[ -f "${destination}" ]] && + printf '%s %s\n' "${expected_sha256}" "${destination}" | + sha256sum -c --status; then + printf 'cache-hit %s\n' "${destination}" + exit 0 +fi + +mkdir -p "$(dirname "${destination}")" +temporary="${destination}.part.${BUILDKITE_BUILD_ID:-local}.$$" +trap 'rm -f "${temporary}"' EXIT +curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ + "${url}" -o "${temporary}" +printf '%s %s\n' "${expected_sha256}" "${temporary}" | sha256sum -c - +chmod 0444 "${temporary}" +mv -f "${temporary}" "${destination}" +trap - EXIT +printf 'cache-fill %s\n' "${destination}" +``` + +- [ ] **Step 4: Run the downloader test and record GREEN** + +Run: `bash builder/downloaders/tests/test-cache-verified-download.sh` + +Expected: PASS with `cache downloader checks passed`. + +- [ ] **Step 5: Commit** + +```bash +git add builder/downloaders/cache-verified-download.sh builder/downloaders/tests/test-cache-verified-download.sh +git commit -s -m "build(cache): add verified archive cache primitive" -m "Publish immutable downloads only after checksum verification so jobs can safely reuse a PVC without treating it as an authority." +``` + +### Task 2: Build driver and initial Kata-stage integration (historical baseline) + +> **Historical-phase note:** The steps below describe the initial Kata-only +> delivery that established the PVC and named-context mechanism. They are not +> the current operational contract. The completed extension consumes the same +> `akernel-download-cache` named context read-only in the Kata, gVisor, and +> OpenTelemetry Docker consumers. `deploy/scripts/build-image.sh` now +> prefetches all three artifacts and forwards their versions, URLs, and pinned +> digests; the gVisor and OpenTelemetry version/digest overrides are required +> pairs. + +**Files:** +- Modify: `deploy/scripts/tests/test-build-image-rrt.sh` +- Modify: `deploy/scripts/build-image.sh` +- Modify: `builder/node.Dockerfile` + +**Interfaces:** +- Consumes: optional `AKERNEL_DEPENDENCY_CACHE_DIR` and Task 1. +- Historical baseline output: named context + `akernel-download-cache=` mounted read-only in the Kata stage. +- Current output: that named context is mounted read-only in the Kata, gVisor, + and OpenTelemetry consumers; Docker receives `KATA_*`, `GVISOR_*`, and + `OTELCOL_CONTRIB_*` versions, URLs, and pinned digest arguments. + +- [ ] **Step 1: Write failing build contract assertions** + +The historical fixture first asserted the Kata arguments below. The current +fixture also asserts gVisor `release-20260706.0` with its SHA-512 digest and +OpenTelemetry Collector contrib `0.120.0` with its SHA-256 digest, all three +deterministic cache paths, three cache-fill/cache-hit outcomes, and read-only +named-context mounts in every consumer. + +The original Kata baseline asserted that the node Docker invocation contains: + +```text +--build-context akernel-download-cache= +--build-arg KATA_RELEASE=4.0.0 +--build-arg KATA_AMD64_SHA256=2c3b9dfeba355582b40aee462b12916c9740654d0230f696adf719d67b063a8c +``` + +The current Dockerfile contract requires the same read-only mount, cache-hit +branch, and digest check after the hit/miss branch for Kata (SHA-256), gVisor +(SHA-512), and OpenTelemetry (SHA-256). + +- [ ] **Step 2: Run the contract test and record RED** + +Run: `bash deploy/scripts/tests/test-build-image-rrt.sh` + +Historical expected result: FAIL because the context and mount did not exist. +The current behavior contract instead requires all three consumers to expose +the read-only mount and their matching digest validation. + +- [ ] **Step 3: Wire deterministic prefetch into the build driver** + +The initial phase added defaults matching the Dockerfile for +`KATA_RELEASE=4.0.0`, +`KATA_AMD64_SHA256=2c3b9dfeba355582b40aee462b12916c9740654d0230f696adf719d67b063a8c`, +and `KATA_RELEASE_BASE_URL=https://github.com/kata-containers/kata-containers/releases/download`. +When Kata and the cache are enabled, it called Task 1 with: + +```bash +kata_filename="kata-static-${kata_release}-amd64.tar.zst" +kata_cache_path="${dependency_cache_dir}/kata/${kata_release}/amd64/${kata_amd64_sha256}/${kata_filename}" +"${AKERNEL_REPO_ROOT}/builder/downloaders/cache-verified-download.sh" \ + "${kata_release_base_url}/${kata_release}/${kata_filename}" \ + "${kata_amd64_sha256}" "${kata_cache_path}" +``` + +When the variable is unset, the current driver uses an empty `mktemp -d` +directory cleaned on exit and does not call the host downloader. Before +invoking Docker, it requires `docker build --help` to contain `--build-context` +and fails with `Docker BuildKit named-context support is required` otherwise. +It passes `--build-context akernel-download-cache=` and forwards the Kata, +gVisor, and OpenTelemetry versions, URLs, and digests. The gVisor and +OpenTelemetry version/digest arguments must be overridden together. + +- [ ] **Step 4: Consume the cache in the Docker stage** + +The historical Kata stage mounted the named context read-only, computed the +same deterministic path, copied a present cache file to the stage-local +archive, otherwise executed the existing `curl`, then ran `sha256sum -c` after +either branch. The current Dockerfile repeats that pattern for all three +artifacts, using `sha512sum -c` for gVisor and retaining the gVisor installation +path plus OpenTelemetry configuration and systemd wiring. + +The exact shell branch is: + +```bash +cache_archive="/var/cache/akernel-downloads/kata/${KATA_RELEASE}/amd64/${KATA_AMD64_SHA256}/kata-static-${KATA_RELEASE}-amd64.tar.zst"; \ +if [ -f "${cache_archive}" ]; then \ + echo "kata-cache-hit ${cache_archive}"; \ + cp "${cache_archive}" "${archive}"; \ +else \ + curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ + "${KATA_RELEASE_BASE_URL}/${KATA_RELEASE}/kata-static-${KATA_RELEASE}-amd64.tar.zst" \ + -o "${archive}"; \ +fi; \ +echo "${KATA_AMD64_SHA256} ${archive}" | sha256sum -c -; +``` + +- [ ] **Step 5: Run focused tests and record GREEN** + +Run: + +```bash +bash builder/downloaders/tests/test-cache-verified-download.sh +bash deploy/scripts/tests/test-build-image-rrt.sh +``` + +Expected current result: both PASS; the first cached fixture build downloads +Kata, gVisor, and OpenTelemetry once, the second reports all three cache hits, +and uncached mode leaves host curl unused. + +- [ ] **Step 6: Commit** + +```bash +git add builder/node.Dockerfile deploy/scripts/build-image.sh deploy/scripts/tests/test-build-image-rrt.sh +git commit -s -m "build(kata): consume verified dependency cache" -m "Expose the checksum-pinned static archive to BuildKit while preserving the upstream fallback." +``` + +This is the historical baseline commit. The later three-artifact integration +is recorded separately and is the authoritative implementation described by +the Global Constraints and operational README above. + +### Task 3: Buildkite PVC isolation + +**Files:** +- Modify: `.buildkite/tests/test_pipeline.py` +- Modify: `.buildkite/pipeline.sh` +- Create: `.buildkite/kubernetes/akernel-dependency-cache-pvc.yaml` +- Modify: `.buildkite/README.md` + +**Interfaces:** +- Consumes: claim `akernel-dependency-cache`. +- Produces: `AKERNEL_DEPENDENCY_CACHE_DIR=/var/cache/akernel-downloads` only in `build-image`. + +- [ ] **Step 1: Write failing PodSpec assertions** + +Assert the build PodSpec has a `dependency-cache` PVC volume and only `container-0` mounts it at `/var/cache/akernel-downloads`. Assert the build environment exports the directory. Assert the resolve and package PodSpecs have no cache volume, mount, or environment variable. + +- [ ] **Step 2: Run the pipeline test and record RED** + +Run: `python3 -m unittest .buildkite/tests/test_pipeline.py -v` + +Expected: FAIL because `dependency-cache` is absent. + +- [ ] **Step 3: Add image-job-only wiring** + +Add only to the image PodSpec: + +```yaml +- name: dependency-cache + persistentVolumeClaim: + claimName: akernel-dependency-cache +``` + +Mount it only into `container-0`; do not add it to `extraVolumeMounts`, which would expose it to checkout. + +- [x] **Step 4: Add manifest and documentation** + +Create a namespace-neutral 10 GiB `ReadWriteOnce` PVC manifest. Cluster +inspection found that Guiyang has no default StorageClass and its Everest +`csi-local-topology` class has no `persistent` local-volume pool on the amd64 +builders. Define a static local PV for the existing +`/mnt/paas/build-cache/akernel-dependency-cache` host path, a +`kubernetes.io/no-provisioner` class with `WaitForFirstConsumer` and `Retain`, +and the claim. Document node replacement maintenance, that cache entries are +disposable, and that every hit is verified. + +- [ ] **Step 5: Run GREEN checks** + +Run: + +```bash +python3 -m unittest .buildkite/tests/test_pipeline.py -v +bash -n .buildkite/pipeline.sh +``` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add .buildkite/pipeline.sh .buildkite/tests/test_pipeline.py .buildkite/kubernetes/akernel-dependency-cache-pvc.yaml .buildkite/README.md +git commit -s -m "ci(buildkite): persist verified Kata downloads" -m "Mount a narrow dependency cache only in the image job so repeated builds avoid downloading immutable archives." +``` + +### Task 4: Local verification and publication + +**Files:** Verify only. + +**Interfaces:** +- Consumes: Tasks 1-3. +- Produces: clean pushed feature branch. + +- [ ] **Step 1: Run all gates** + +```bash +bash builder/downloaders/tests/test-cache-verified-download.sh +bash deploy/scripts/tests/test-build-image-rrt.sh +make SHELL=/bin/bash buildkite-check +make SHELL=/bin/bash deploy-script-check +git diff --check +``` + +Expected: zero failures and silent `git diff --check`. + +- [ ] **Step 2: Review scope and secrets** + +Run `git status --short`, `git diff HEAD~3 --stat`, and `git log --format='%h %s%n%b' -4`. Expected: only planned files; no Secret, token, kubeconfig, presigned URL, or cache content. + +- [ ] **Step 3: Push the existing feature branch** + +Run: `git push chamberlain codex/yuanrong-downloaders` + +Expected: remote advances to the verified commits. + +### Task 5: Cold-fill, warm-hit, and formal acceptance + +**Files:** +- Apply: `.buildkite/kubernetes/akernel-dependency-cache-pvc.yaml` +- Save evidence outside Git: `/Users/chamberlain/.codex/evidence/akernel-image-pvc-cache-20260814/` + +**Interfaces:** +- Consumes: Guiyang Buildkite namespace and `akernel-image`. +- Produces: bound PVC, cold-fill build, warm-hit build, image/deployment/SDK artifacts, sanitized evidence. + +- [ ] **Step 1: Discover namespace and default storage class read-only** + +Using authorized CCE access or a non-secret Buildkite diagnostic, run `kubectl get storageclass`, `kubectl get pvc -A`, and `kubectl auth can-i create persistentvolumeclaims`. Record only non-secret storage metadata. + +- [ ] **Step 2: Apply and verify the claim** + +Apply the manifest in the Buildkite job namespace and run `kubectl get pvc akernel-dependency-cache -o wide`. Expected: `Bound`, or `Pending` only for `WaitForFirstConsumer`. + +- [ ] **Step 3: Incrementally update pipeline configuration** + +Re-read the current Buildkite configuration, compare it with `.buildkite/pipeline.yml`, and preserve `checkout.submodules=false`, the WireGuard PodSpec, proxies, `NO_PROXY`, and all current changes. + +- [ ] **Step 4: Run cold-fill formal build** + +Trigger release YuanRong 0.9.7 with Kata/NVIDIA enabled and `standalone,helm`. Verify `cache-fill`, the exact checksum, checkout/WireGuard/proxy requirements, exact submodules, SWR push, image manifest, standalone bundle, Helm bundle, and sandbox SDK. + +- [ ] **Step 5: Run warm-hit formal build** + +Trigger identical inputs again. Verify `cache-hit`, no Kata archive `curl` progress, the same pinned digest, and complete outputs. + +- [ ] **Step 6: Save sanitized evidence** + +Save build JSON, job summaries, sanitized logs, PVC metadata, artifact lists/manifests, and image digest. Strip URL query strings and exclude all credentials and kubeconfigs. + +- [ ] **Step 7: Post-success control-plane cleanup** + +After the warm build passes: archive `akernel-egress-probe-20260814`; tighten `AKERNEL_WG_CONFIG` policy to `akernel-image` only without reading it; verify Squid/security-group restrictions; keep the Hong Kong ECS running. diff --git a/docs/superpowers/plans/2026-08-14-gvisor-otel-pvc-cache.md b/docs/superpowers/plans/2026-08-14-gvisor-otel-pvc-cache.md new file mode 100644 index 0000000..9e6290a --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-gvisor-otel-pvc-cache.md @@ -0,0 +1,256 @@ +# gVisor And OpenTelemetry PVC Cache Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Extend AKernel's verified Buildkite dependency PVC so the pinned gVisor `runsc` binary and OpenTelemetry Collector contrib archive are cached alongside Kata. + +**Architecture:** The existing host downloader accepts either a 64-character SHA-256 or 128-character SHA-512 digest and atomically publishes only verified bytes. The build driver prefetches deterministic gVisor and OpenTelemetry paths when `AKERNEL_DEPENDENCY_CACHE_DIR` is configured, then the final Docker stage consumes the shared named context read-only and independently verifies each pinned digest before installation. + +**Tech Stack:** Bash, Docker BuildKit named contexts, Dockerfile bind mounts, Buildkite Kubernetes PVC, shell contract tests, Python `unittest`. + +## Global Constraints + +- Keep gVisor at `release-20260706.0`; do not change its runtime packaging or installed path. +- Pin amd64 `runsc` SHA-512 to `73938c145ebe554cf61a01da455688f4b732eebdf7b1b635bdef5b195868b363d8cb400e3d92ed1f377b78996805556c247a4849583910cb04e92b156053033e`. +- Keep OpenTelemetry Collector contrib at `0.120.0`; do not change its configuration or systemd wiring. +- Pin `otelcol-contrib_0.120.0_linux_amd64.tar.gz` SHA-256 to `81bf885bc9a86705feb3c113c5a356571390e3601eb651ffcf2b3428f6571adb`. +- Preserve the existing Kata cache path and artifact; never force a Kata redownload for this extension. +- The PVC remains a disposable performance cache. Repository-pinned URLs and digests remain authoritative. +- Docker consumers mount the cache read-only and verify the digest again before installation. +- An unset `AKERNEL_DEPENDENCY_CACHE_DIR` preserves direct-download fallback for all three dependencies. +- `/var/lib/docker` remains a per-job `emptyDir`; do not place Docker, Cargo, Go, apt, or source caches on this PVC. +- Keep the 10 GiB static local PV, node affinity, checkout isolation, WireGuard hook, NO_PROXY list, and exact submodule checkout unchanged. +- Never read, print, regenerate, or broaden access to `AKERNEL_WG_CONFIG`. +- Do not stop or release the Hong Kong egress ECS. + +--- + +### Task 1: SHA-512 Support In The Atomic Downloader + +**Files:** +- Modify: `builder/downloaders/tests/test-cache-verified-download.sh` +- Modify: `builder/downloaders/cache-verified-download.sh` + +**Interfaces:** +- Consumes: `cache-verified-download.sh URL DIGEST DESTINATION`. +- Produces: verified SHA-256 or SHA-512 cache entries with the existing `cache-fill` and `cache-hit` output contract. + +- [ ] **Step 1: Write the failing SHA-512 behavior test** + +Add a second fixture invocation using: + +```bash +expected_sha512="$(sha512sum "${fixture}" | awk '{print $1}')" +sha512_file="${test_root}/cache/gvisor/release-test/x86_64/${expected_sha512}/runsc" +sha512_output="$("${DOWNLOADER}" \ + https://example.invalid/runsc "${expected_sha512}" "${sha512_file}")" +[[ "${sha512_output}" == "cache-fill ${sha512_file}" ]] +cmp "${fixture}" "${sha512_file}" +``` + +Also assert a 96-character lowercase hexadecimal digest is rejected before fake curl is called. + +- [ ] **Step 2: Run the test and verify RED** + +Run: `bash builder/downloaders/tests/test-cache-verified-download.sh` + +Expected: FAIL because the downloader accepts only 64-character SHA-256 digests. + +- [ ] **Step 3: Implement digest-length dispatch** + +Select the checker without changing the three-argument interface: + +```bash +case "${expected_digest}" in + (*[!0-9a-f]*) die "cache digest must use lowercase hexadecimal characters" ;; +esac +case "${#expected_digest}" in + 64) checksum_command=sha256sum ;; + 128) checksum_command=sha512sum ;; + *) die "cache digest must be a SHA-256 or SHA-512 hexadecimal value" ;; +esac +``` + +Use `"${checksum_command}" -c -` for hit and temporary-file validation. Preserve unique same-directory temporary files, atomic rename, permissions, cleanup, and output text. + +- [ ] **Step 4: Run GREEN verification** + +Run: `bash builder/downloaders/tests/test-cache-verified-download.sh` + +Expected: PASS with `cache downloader checks passed`. + +- [ ] **Step 5: Commit the downloader change** + +```bash +git add builder/downloaders/cache-verified-download.sh builder/downloaders/tests/test-cache-verified-download.sh +git commit -s -m "build(cache): verify SHA-512 artifacts" -m "Allow the shared atomic cache primitive to validate the official gVisor digest without weakening existing SHA-256 entries." +``` + +### Task 2: Prefetch And Read-Only Docker Consumption + +**Files:** +- Modify: `deploy/scripts/tests/test-build-image-rrt.sh` +- Modify: `deploy/scripts/build-image.sh` +- Modify: `builder/node.Dockerfile` +- Modify: `Makefile` + +**Interfaces:** +- Consumes: `GVISOR_RELEASE`, `GVISOR_AMD64_SHA512`, `GVISOR_RELEASE_BASE_URL`, `OTELCOL_CONTRIB_VERSION`, `OTELCOL_CONTRIB_SHA256`, and optional `OTELCOL_CONTRIB_URL`. +- Produces: deterministic cache entries and Docker build arguments for both dependencies. + +- [ ] **Step 1: Write failing build-driver behavior assertions** + +Extend the behavior fixture with one byte source shared by fake curl and derive both digest algorithms: + +```bash +gvisor_sha512="$(sha512sum "${dependency_source}" | awk '{print $1}')" +otel_sha256="$(sha256sum "${dependency_source}" | awk '{print $1}')" +``` + +Run the cached build with test versions and URLs, then assert these exact entries exist: + +```text +gvisor/release-9.9.9/x86_64//runsc +otelcol-contrib/8.8.8/linux-amd64//otelcol-contrib_8.8.8_linux_amd64.tar.gz +``` + +The first build must invoke fake curl three times total for Kata, gVisor, and OpenTelemetry. The second build must report three `cache-hit` lines and leave the count at three. The uncached build must not invoke host curl. + +Assert the node Docker invocation forwards all versions, URLs, and digests. Add contract checks for `gvisor-cache-hit`, `otelcol-cache-hit`, SHA-512/SHA-256 verification, and read-only `akernel-download-cache` mounts in the final stage. + +- [ ] **Step 2: Run the contract test and verify RED** + +Run: `bash deploy/scripts/tests/test-build-image-rrt.sh` + +Expected: FAIL because only Kata is prefetched and only the Kata stage consumes the named context. + +- [ ] **Step 3: Add validated defaults and override pairs** + +In `deploy/scripts/build-image.sh`, define the exact current defaults and validate release/version syntax, 128-character gVisor SHA-512, 64-character OpenTelemetry SHA-256, and HTTP(S) URLs. Track whether each version and digest was explicitly overridden; reject a version-only or digest-only override before Docker starts. + +Expose `--gvisor-amd64-sha512`, `--otelcol-contrib-version`, `--otelcol-contrib-sha256`, and `--otelcol-contrib-url`. Forward matching Make variables from `Makefile`, including help text that shows the version/digest pair requirement. + +- [ ] **Step 4: Prefetch deterministic host entries** + +When the real PVC cache is configured, invoke the verified downloader for: + +```bash +gvisor_cache_path="${dependency_cache_dir}/gvisor/${gvisor_release}/x86_64/${gvisor_amd64_sha512}/runsc" +otel_filename="otelcol-contrib_${otelcol_contrib_version}_linux_amd64.tar.gz" +otel_cache_path="${dependency_cache_dir}/otelcol-contrib/${otelcol_contrib_version}/linux-amd64/${otelcol_contrib_sha256}/${otel_filename}" +``` + +Build the gVisor URL by stripping the required `release-` prefix. Build the default OpenTelemetry URL from its version unless an exact mirror URL is supplied. + +- [ ] **Step 5: Consume gVisor from the read-only named context** + +Add `ARG GVISOR_AMD64_SHA512`, mount `akernel-download-cache` read-only on the gVisor `RUN`, copy an exact cached `runsc` when present, otherwise download from the existing official URL, then verify with: + +```bash +echo "${GVISOR_AMD64_SHA512} runsc" | sha512sum -c - +``` + +Emit `gvisor-cache-hit ` on a hit and retain the existing architecture validation and `/usr/local/bin/runsc` installation. + +- [ ] **Step 6: Consume OpenTelemetry from the read-only named context** + +Add `ARG OTELCOL_CONTRIB_VERSION` and `ARG OTELCOL_CONTRIB_SHA256`. Mount the named context read-only, copy the exact cached archive or download the existing URL, verify SHA-256, extract only `otelcol-contrib`, preserve mode `0755`, and emit `otelcol-cache-hit ` on a hit. + +- [ ] **Step 7: Run focused GREEN verification** + +Run: + +```bash +bash builder/downloaders/tests/test-cache-verified-download.sh +bash deploy/scripts/tests/test-build-image-rrt.sh +``` + +Expected: both PASS; first fake build downloads three artifacts, second downloads none, and uncached mode uses only Docker fallback. + +- [ ] **Step 8: Commit build integration** + +```bash +git add Makefile builder/node.Dockerfile deploy/scripts/build-image.sh deploy/scripts/tests/test-build-image-rrt.sh +git commit -s -m "build(cache): reuse gVisor and OTel artifacts" -m "Prefetch checksum-pinned node dependencies into the existing PVC and revalidate them from a read-only BuildKit context." +``` + +### Task 3: Documentation And Full Local Gate + +**Files:** +- Modify: `.buildkite/README.md` +- Modify: `docs/superpowers/plans/2026-08-14-buildkite-large-dependency-pvc-cache.md` + +**Interfaces:** +- Consumes: Tasks 1-2 behavior and exact pinned artifacts. +- Produces: operational guidance and a complete verified feature branch. + +- [ ] **Step 1: Update the operational cache inventory** + +Document all three component paths, algorithms, digests, sizes, version/digest override pairing, corrupt-entry replacement, and that 10 GiB remains sufficient. Replace the earlier audit statements that deferred gVisor and OpenTelemetry. + +- [ ] **Step 2: Run the full local quality gate** + +Run: + +```bash +bash builder/downloaders/tests/test-cache-verified-download.sh +bash deploy/scripts/tests/test-build-image-rrt.sh +make SHELL=/bin/bash buildkite-check +make SHELL=/bin/bash deploy-script-check +git diff --check +``` + +Expected: zero failures and silent `git diff --check`. + +- [ ] **Step 3: Review scope and secrets** + +Verify `git status --short`, inspect the complete diff, and scan the branch diff for cloud credentials, WireGuard configuration, API tokens, kubeconfig data, and presigned URLs. Only planned source, tests, and documentation may be committed. + +- [ ] **Step 4: Commit and push** + +```bash +git add .buildkite/README.md docs/superpowers/plans/2026-08-14-buildkite-large-dependency-pvc-cache.md docs/superpowers/plans/2026-08-14-gvisor-otel-pvc-cache.md +git commit -s -m "docs(buildkite): document expanded artifact cache" -m "Record the pinned gVisor and OpenTelemetry entries and the operational rules for safely reusing them." +git push chamberlain codex/yuanrong-downloaders +``` + +### Task 4: Formal Cold-Fill And Warm-Hit Acceptance + +**Files:** +- Save evidence outside Git under `/Users/chamberlain/.codex/evidence/akernel-gvisor-otel-cache-20260814/`. + +**Interfaces:** +- Consumes: pushed exact feature commit, bound `akernel-dependency-cache` PVC, and the existing `akernel-image` pipeline. +- Produces: two passed formal builds and sanitized acceptance evidence. + +- [ ] **Step 1: Trigger the first formal build** + +Use the exact pushed commit with release YuanRong 0.9.7, `standalone,helm`, Kata enabled, NVIDIA enabled, and the current SWR repository. Do not delete or invalidate the existing Kata entry. The new gVisor and OpenTelemetry paths should fill naturally on their first use. + +- [ ] **Step 2: Verify the first build** + +Require all jobs to pass and confirm: + +```text +cache-hit .../kata/4.0.0/amd64/... +cache-fill .../gvisor/release-20260706.0/x86_64/.../runsc +cache-fill .../otelcol-contrib/0.120.0/linux-amd64/...tar.gz +gvisor-cache-hit .../runsc +otelcol-cache-hit ...tar.gz +runsc: OK +otelcol-contrib_0.120.0_linux_amd64.tar.gz: OK +``` + +Also verify the image digest and standalone, Helm, sandbox SDK, manifests, and SHA256SUMS artifacts. + +- [ ] **Step 3: Trigger an identical warm build** + +Use the same commit and environment. Require host `cache-hit` for Kata, gVisor, and OpenTelemetry, Docker hit markers for all three, successful independent digest checks, and no host `cache-fill`. + +- [ ] **Step 4: Recheck egress and controls** + +Confirm checkout hook ordering, recent WireGuard handshake with bidirectional traffic, GitHub success, only the expected `example.com` 403, no recursive submodule checkout, successful SWR/artifact traffic, Secret policy still restricted to `akernel-image`, probe pipeline still archived, and Hong Kong ECS still ACTIVE with no inbound 3128 rule. + +- [ ] **Step 5: Save sanitized evidence and report timings** + +Save filtered job logs, build/artifact JSON, image manifests, digest markers, cache fill/hit timestamps, final Secret/probe/ECS readbacks, and local test output. Scan all evidence for secrets and presigned query strings. Compare first and second image-job timing and identify remaining uncached bottlenecks without broadening the PVC scope. diff --git a/docs/superpowers/plans/2026-08-14-openyuanrong-downloaders.md b/docs/superpowers/plans/2026-08-14-openyuanrong-downloaders.md new file mode 100644 index 0000000..4a692ce --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-openyuanrong-downloaders.md @@ -0,0 +1,126 @@ +# Replaceable openYuanRong Downloaders Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Move core-wheel and RRT-runtime artifact acquisition behind two replaceable scripts while preserving public Release behavior and existing URL/SHA overrides. + +**Architecture:** Artifact-specific scripts own source selection, transport, checksum validation, and RRT wheel extraction. Dockerfiles copy and execute those scripts, then validate only the physical artifact contract required by the image. Pipeline users replace the scripts in the build context before `docker build` to select private OBS behavior. + +**Tech Stack:** Bash 3.2-compatible shell scripts, Dockerfiles, curl, SHA-256, unzip, existing AKernel shell contract tests. + +## Global Constraints + +- Preserve `OPEN_YR_CORE_WHEEL_URL/SHA256` and `OPEN_YR_RRT_WHEEL_URL/SHA256` as all-or-nothing compatibility pairs. +- Preserve pinned Release URLs, versions, checksums, core installation layout, and x86-64 ELF validation. +- Do not modify the current pause/resume patch-selection behavior. +- New downloader paths must be stable files in the Docker build context and require no buildx-only feature. +- Do not include unrelated dirty files from the primary AKernel checkout. + +--- + +### Task 1: Specify and implement downloader behavior + +**Files:** +- Create: `builder/downloaders/tests/test-openyuanrong-downloaders.sh` +- Create: `builder/downloaders/download-openyuanrong-core.sh` +- Create: `builder/downloaders/download-openyuanrong-rrt.sh` + +**Interfaces:** +- Consumes: existing `OPEN_YR_VERSION`, Release URL/checksum variables, URL/SHA override pairs, and `TARGETARCH`. +- Produces: `download-openyuanrong-core.sh DEST_DIR` with exactly one wheel; `download-openyuanrong-rrt.sh DEST_FILE` with one raw runtime. + +- [ ] **Step 1: Write the failing downloader test** + +Create local `file://` fixtures with literal content and hand-computed SHA-256 values. Exercise Release core, encoded OBS core basename, Release raw RRT, OBS RRT wheel, checksum mismatch, and missing RRT member. Assert final file content and that failure leaves no published output. + +- [ ] **Step 2: Run the test to verify RED** + +Run: `bash builder/downloaders/tests/test-openyuanrong-downloaders.sh` + +Expected: FAIL because both downloader scripts do not exist. + +- [ ] **Step 3: Implement the minimal core downloader** + +Use `set -euo pipefail`, validate the destination argument and URL/SHA pair, resolve architecture, derive or decode the wheel basename, download into a temporary directory, verify SHA-256, then atomically move the wheel into the destination directory. + +- [ ] **Step 4: Implement the minimal RRT downloader** + +Use the same pair validation and temporary-directory cleanup. Download and verify either the Release raw binary or override wheel; for a wheel, use `unzip -p ... openyuanrong_rrt/rrt-runtime` into a temporary output and move it to the requested destination only after extraction succeeds. + +- [ ] **Step 5: Run the downloader test to verify GREEN** + +Run: `bash builder/downloaders/tests/test-openyuanrong-downloaders.sh` + +Expected: PASS with every fixture case succeeding and both negative cases rejected. + +### Task 2: Replace inline Dockerfile acquisition + +**Files:** +- Modify: `builder/node.Dockerfile` +- Modify: `builder/runtime.Dockerfile` +- Modify: `deploy/scripts/tests/test-build-image-rrt.sh` + +**Interfaces:** +- Consumes: Task 1 downloader contracts. +- Produces: Docker stages that execute replaceable scripts and enforce artifact-shape validation. + +- [ ] **Step 1: Add failing Dockerfile contract assertions** + +Extend the existing build contract test to require both downloader COPY/invocation paths, require core single-wheel and RRT ELF validation, and reject the old inline core URL construction and RRT download branches. + +- [ ] **Step 2: Run the contract test to verify RED** + +Run: `bash deploy/scripts/tests/test-build-image-rrt.sh` + +Expected: FAIL because neither Dockerfile copies the new downloader paths. + +- [ ] **Step 3: Wire the core downloader** + +Copy `builder/downloaders/download-openyuanrong-core.sh` into the node image before installation. Execute it with an empty temporary directory, use a shell glob plus `test -f` to require exactly one wheel, keep the existing pip installation and `yr` checks, and remove the temporary script/artifact after installation. + +- [ ] **Step 4: Wire the RRT downloader** + +Copy `builder/downloaders/download-openyuanrong-rrt.sh` into the `rrt-download` stage. Replace both inline source branches with one script invocation and retain `chmod`, `file`, architecture validation, and final stage copy. + +- [ ] **Step 5: Run focused tests to verify GREEN** + +Run: + +```bash +bash deploy/scripts/tests/test-build-image-rrt.sh +bash builder/downloaders/tests/test-openyuanrong-downloaders.sh +``` + +Expected: both PASS. + +### Task 3: Document and verify the supported override boundary + +**Files:** +- Modify: `AGENTS.md` + +**Interfaces:** +- Consumes: the stable downloader paths from Tasks 1 and 2. +- Produces: maintainer guidance for public Release builds and private pipeline replacement. + +- [ ] **Step 1: Update build documentation** + +Document that default scripts remain checksum-pinned, list both replaceable paths, define each output contract, and state that a private replacement owns source authentication and checksum validation. + +- [ ] **Step 2: Run full relevant verification** + +Run: + +```bash +bash builder/downloaders/tests/test-openyuanrong-downloaders.sh +bash deploy/scripts/tests/test-build-image-rrt.sh +make SHELL=/bin/bash deploy-script-check +docker build --check -f builder/runtime.Dockerfile . +docker build --check -f builder/node.Dockerfile . +git diff --check +``` + +Expected: all commands exit 0. If Docker `--check` is unavailable or the daemon is unavailable, report that environmental limitation and retain the shell/Dockerfile contract results. + +- [ ] **Step 3: Commit implementation** + +Commit the scripts, tests, Dockerfiles, and `AGENTS.md` with a signed Conventional Commit describing the replaceable artifact acquisition boundary. diff --git a/docs/superpowers/specs/2026-08-12-standalone-rrt-pause-resume-e2e-design.md b/docs/superpowers/specs/2026-08-12-standalone-rrt-pause-resume-e2e-design.md new file mode 100644 index 0000000..2a49e86 --- /dev/null +++ b/docs/superpowers/specs/2026-08-12-standalone-rrt-pause-resume-e2e-design.md @@ -0,0 +1,468 @@ +# AKernel Standalone RRT Pause/Resume 端到端验证设计 + +| 字段 | 值 | +|---|---| +| 编号 | AKERNEL-PR-E2E-001 | +| 状态 | 草案 | +| 作者 | ChamberlainJI、Codex | +| SIG / 模块 | AKernel Builder、Standalone、YuanRong Sandbox | +| 评审人 | AKernel 与 YuanRong pause/resume 维护者 | +| 批准人 | AKernel 仓库维护者 | +| 创建日期 | 2026-08-12 | + +## 摘要 + +本方案为 AKernel all-in-one 镜像增加条件式 x86_64 RRT runtime 能力，并在 standalone +模式显式启用 YuanRong pause/resume 数据面，使 `openyuanrong-sandbox` SDK 能从 +`Sandbox.pause()` / `Sandbox.resume()` 一路验证到 FunctionSystem、DataSystem、 +sandboxd 和 runsc。默认镜像与默认启动行为保持不变；首轮验证固定为单节点、runsc 和 +DataSystem，并以 pause 前后台进程在 resume 后继续运行为核心成功证据。 + +## 背景与动机 + +AKernel 当前 builder 只生成四套 Python runtime rootfs，service metadata 中没有 RRT； +当前 `src/sandboxd` gitlink也不包含 Checkpoint/Restore RPC。用户已经通过 Buildkite #215 +构建出包含 pause/resume 功能的 YuanRong core、RRT 和 Sandbox SDK，但实物核验确认该 +core wheel 的 process-mode 脚本没有暴露 feature gate、snapshot backend 和共享 +checkpoint root。若只替换 core wheel，standalone 的 pause/resume 仍保持默认关闭，且 +默认 checkpoint 路径不在 sandboxd 管理根下。本设计用于补齐这些可验证的集成缺口， +形成从公开 SDK 到恢复后进程连续性的完整闭环。 + +### 目标 + +- 在 Linux x86_64 上用 checksum-pinned #215 core 与 RRT 构建 AKernel all-in-one 镜像。 +- 默认不广告缺失的 RRT service，也不默认开启 pause/resume feature gate。 +- 显式启用后，让 FunctionAgent、RuntimeManager 与 sandboxd 使用同一受控 checkpoint + root，并用真实 DataSystem SnapshotStorage 完成上传与下载。 +- 从 `yr_sandbox.Sandbox` 完成两轮 pause/resume，并证明相同 sandbox ID、文件内容和 + pause 前进程均连续。 +- 产出可关联 SDK request/snapshot ID、控制面日志、DataSystem 和 runsc 的脱敏证据。 + +### 非目标 + +- 不在首轮验证 Kubernetes/CCE、多节点迁移或跨节点 Resume。 +- 不在首轮验证 OBS、进程重启后的持久化恢复或 snapshot GC。 +- 不宣称 pause/resume 已达到生产就绪状态。 +- 不修改 `akernel_sdk.Sandbox` 公共 API；本轮客户端入口是 YuanRong 的 + `yr_sandbox.Sandbox`。 +- 不把 RRT 与 Rust FunctionSystem 混为一谈。控制面继续使用 Buildkite #215 的 C++ + FunctionSystem；RRT 仅指 sandbox 内运行的 Rust runtime daemon。 + +## 方案概述 + +Linux x86_64 操作者向现有 `make build` 同时提供 core wheel 与 RRT wheel 的 URL/SHA， +builder 校验并把 RRT 安装进 EROFS rootfs，再选择包含 `rrt` slot 的 service metadata。 +node image 对 #215 process 脚本应用上下文严格匹配的兼容 patch。镜像启动时只有 +`AKERNEL_ENABLE_PAUSE_RESUME=true` 才向 YuanRong 传 gate、DataSystem backend 和 +sandboxd 管理根下的 checkpoint path。宿主机 E2E runner 安装同一 Buildkite 构建的 +Sandbox SDK，执行下面的完整数据流： + +```text +openyuanrong-sandbox SDK（宿主机） + -> Frontend HTTP API + -> FunctionProxy（合并 FunctionAgent/RuntimeManager） + -> sandboxd Checkpoint/Restore RPC + -> runsc checkpoint/restore + -> DataSystem SnapshotStorage + -> RRT 恢复并重新建立 RuntimeRPC/直连路由 +``` + +该方案遵循 YuanRong `2026-08-10-pause-resume-agent-data-plane-design.md` 的同版本组件、 +默认关闭 feature gate、真实 SnapshotStorage、确定性 Resume 和可追踪诊断原则。 + +### 用户故事 + +作为 pause/resume 功能开发者，我希望在一台普通 x86 Linux 服务器上用同一组 Buildkite +artifact 构建并启动 AKernel，然后从 Sandbox SDK 发起 pause/resume，从而判断完整链路 +是否保持运行中 agent 工作负载的进程和文件状态。 + +### 约束与注意事项 + +- 镜像构建和 runsc checkpoint/restore 只能在 privileged Linux x86_64 环境验证； + macOS 只检查源码、builder 脚本、wheel、checksum 和 ELF header。 +- RRT wheel 是 Python ABI 无关的原生二进制载体，不属于 cp310/cp311/cp312/cp313 SDK + wheel 集合。 +- #215 先用于 bring-up；若命中其后的 FunctionSystem teardown 修复，必须整组重建， + 不能混用不同顶层提交的二进制。 + +### 风险与缓解措施 + +| 风险 | 缓解措施 | +|---|---| +| #215 process 脚本未接线导致 feature 实际未开启 | compatibility patch 上下文不匹配即构建失败；启动后检查真实进程 argv | +| checkpoint path 越过 sandboxd 管理根 | 固定为 `/home/akernel/sandboxd/root/checkpoints`，禁止外部覆盖 | +| 默认镜像误广告 RRT | 只有 RRT wheel URL/SHA 成对存在时选择 RRT services 配置 | +| #215 缺少后续 self-await 修复 | bring-up 记录问题；最终通过前从同一新顶层提交整组重建和复测 | +| 诊断包泄露 token 或云凭据 | 只收集允许列表中的日志和元数据，并在落盘前脱敏 | + +## 详细设计 + +### 已核验基线 + +#### Buildkite #215 产物 + +构建链接： + +| 产物 | URL | SHA-256 | 用途 | +| --- | --- | --- | --- | +| `openyuanrong-core` | `https://openyuanrong.obs.cn-southwest-2.myhuaweicloud.com/daily/20260811190323/linux/amd64/openyuanrong_core-0.7.0%2B6dfa49681774-py3-none-manylinux_2_31_x86_64.whl` | `39ba1cf8323ac4e784117867ecd806ec392da05aa2fd87130f8830eb56310895` | Frontend、C++ FunctionSystem、DataSystem、FaaS 控制面 | +| `openyuanrong-rrt` | `https://openyuanrong.obs.cn-southwest-2.myhuaweicloud.com/daily/20260811184249/linux/amd64/openyuanrong_rrt-0.7.0%2B6dfa49681774-py3-none-manylinux_2_31_x86_64.whl` | `51e16a48a98ff89497e268e939ae046205c9ba287e8b0b571eec3048e6d38ae7` | rootfs 内的 x86_64 `rrt-runtime` | +| `openyuanrong-sandbox` | `https://openyuanrong.obs.cn-southwest-2.myhuaweicloud.com/daily/20260811184237/linux/noarch/openyuanrong_sandbox-0.10.1.dev36-py3-none-any.whl` | `ef8985f11d3189fe9d70866614b3a8c3060bc62450bd0a3379e9117599aac190` | 主机侧 E2E 客户端 | + +三项产物均来自 YuanRong 顶层提交 `6dfa49681774`。#215 中对应子模块为： + +- Frontend `d00ba4b7f9036a299d35850b0a299c8777996afb` +- FunctionSystem `6fac483a20edabb21d0f1e48789d02ec11c7c9bc` +- Sandbox SDK `9176fcfdd11bff4c7c59d408bf7602f6db05c1d7` + +#215 的 job 是 `Build X86`、`Build RRT amd64` 和 `Build Runtime amd64 cp311`，没有 +RRT arm64 或 ARM runtime job。本设计的目标平台因此明确限定为原生 Linux x86_64；不把 +Apple Silicon 上的 Docker/QEMU 模拟结果作为镜像构建或 E2E 通过证据。 + +RRT wheel 已验证为 `py3-none-manylinux_2_31_x86_64`，内部仅携带 Python 辅助模块和 +原生 `rrt-runtime`。它不使用 CPython ABI，因此不需要为 cp310、cp312、cp313 分别 +重建 wheel。现有四套 Python runtime 保持不变；RRT 作为第五个独立 service slot 加入。 + +#### 已发现的 process-mode 缺口 + +#215 core wheel 内实际打包的是旧式静态链接 `yr` CLI。它把未知参数继续传给 +`deploy/process/config.sh`，但 #215 的 process 脚本尚未声明或传递以下配置： + +- `enable_sandbox_pause_resume` +- `snapshot_storage_backend` +- 可覆盖的 `checkpoint_dir` + +同时，#215 `functionsystem/deploy/install.sh` 只给独立 FunctionAgent/RuntimeManager 传 +默认 checkpoint root，没有给 standalone 使用的合并 FunctionProxy 传上述参数。因此， +仅替换 core wheel 和添加 RRT 后，pause/resume 仍会因 feature gate 默认关闭而不可用。 + +这个缺口只影响 process/standalone 部署接线，不否定 #215 已编译出的 Frontend、 +FunctionSystem、RRT 和 SDK 功能。 + +#### sandboxd 基线 + +AKernel 当前 `src/sandboxd` gitlink 尚不包含 Checkpoint/Restore RPC。实现阶段应将该 +gitlink 更新到已验证的 checkpoint 分支提交，而不是从同级未跟踪的 `sandboxd/` 工作树 +复制文件。目标实现必须保留以下契约： + +- `CheckpointRequest`: `sandbox_id=1`, `checkpoint_dir=2` +- `RestoreRequest`: `config=1`, `checkpoint_dir=2` +- `RestoreResponse`: `sandbox_id=1` +- 只允许 `runsc` checkpoint/restore +- checkpoint 目录必须位于 sandboxd 管理根之下 + +### 构建设计 + +#### 新增构建输入 + +在 `Makefile`、`deploy/scripts/build-image.sh` 和 Docker build args 中新增成对输入： + +- `OPEN_YR_RRT_WHEEL_URL` +- `OPEN_YR_RRT_WHEEL_SHA256` + +规则如下： + +1. 两者必须同时为空或同时非空。 +2. URL 必须解析为 `.whl` 文件名。 +3. 下载后先校验 SHA-256，再解包/安装。 +4. 当前仅接受 `linux/amd64`；非 amd64 构建在 RRT 输入非空时立即失败。 +5. 是否携带 RRT 是镜像构建能力；是否开启 pause/resume 是运行时开关，两者不能合并成 + 一个默认开启的行为。 + +#### runtime rootfs + +`builder/runtime.Dockerfile` 在收到 RRT 输入时： + +1. 下载并校验 wheel。 +2. 将 `rrt-runtime` 安装到稳定路径 + `/opt/openyuanrong-rrt/rrt-runtime`。 +3. 校验文件可执行并用 `file`/ELF header 验证目标为 x86_64。 +4. 继续用现有 `mkfs.erofs` 生成单一 `yr-runtime-rootfs.img`。 + +RRT 启动不依赖某个 CPython venv，避免把 RRT 误绑到 py310/py311。Python 仅保留给 +现有 Python service slots 和 rootfs 基础工具。 + +#### service metadata + +新增 RRT 版本的 services 配置，保留现有 `yr_services.yaml` 作为无 RRT 默认值。RRT +配置新增： + +```yaml +rrt: + runtime: rust + rootfs: + runtime: runsc + type: local + path: /home/yuanrong/yr-runtime-rootfs.img + readonly: false + mountpoint: /var/task/code + bootstrap: + type: erofs + root: /home/yuanrong/yr-runtime-rootfs.img + entrypoint: /__yuanrong/usr/bin/tini-static -- /__yuanrong/opt/openyuanrong-rrt/rrt-runtime +``` + +`builder/node.Dockerfile` 只在 RRT wheel 能力存在时选择该配置，避免普通镜像广告一个实际 +不存在的 `default/0-defaultservice-rrt/$latest`。 + +#### #215 process 接线兼容层 + +新增版本化 patch 文件，在 node image 安装完 checksum-pinned core wheel 后、仅对 RRT +能力构建应用。patch 必须精确修改并校验两处上游脚本： + +1. `deploy/process/config.sh` + - 声明、解析、校验并 export gate/backend/checkpoint 参数。 +2. `functionsystem/deploy/install.sh` + - checkpoint root 可由 process 配置覆盖； + - 合并 FunctionProxy 获取 gate、backend、checkpoint root、DataSystem 地址； + - 非合并模式只把这些配置交给 FunctionAgent/RuntimeManager，不泄露 OBS 凭据给 + FunctionProxy。 + +patch 上下文不匹配必须让 Docker build 失败，禁止静默构建一个实际未启用 pause/resume +的镜像。后续整组 YuanRong artifact 原生包含这些接线时，删除兼容 patch 及其条件分支。 + +#### x86 构建入口 + +实现完成后，Linux x86_64 服务器使用一条显式、可复现的构建命令： + +```bash +make build \ + IMAGE_REPOSITORY=akernel-all-in-one \ + IMAGE_TAG=pause-resume-215 \ + OPEN_YR_CORE_WHEEL_URL='https://openyuanrong.obs.cn-southwest-2.myhuaweicloud.com/daily/20260811190323/linux/amd64/openyuanrong_core-0.7.0%2B6dfa49681774-py3-none-manylinux_2_31_x86_64.whl' \ + OPEN_YR_CORE_WHEEL_SHA256='39ba1cf8323ac4e784117867ecd806ec392da05aa2fd87130f8830eb56310895' \ + OPEN_YR_RRT_WHEEL_URL='https://openyuanrong.obs.cn-southwest-2.myhuaweicloud.com/daily/20260811184249/linux/amd64/openyuanrong_rrt-0.7.0%2B6dfa49681774-py3-none-manylinux_2_31_x86_64.whl' \ + OPEN_YR_RRT_WHEEL_SHA256='51e16a48a98ff89497e268e939ae046205c9ba287e8b0b571eec3048e6d38ae7' +``` + +构建前必须初始化递归 submodule；构建输出只产生选定的 +`akernel-all-in-one:pause-resume-215` 引用，不额外创建隐式 alias。 + +### 运行时设计 + +#### 默认关闭 + +standalone 新增单一外部开关： + +```bash +AKERNEL_ENABLE_PAUSE_RESUME=true ./deploy/standalone/start.sh +``` + +默认值为 `false`。`start.sh` 把该值传入 all-in-one 容器，`yuanrong.service` 明确允许该 +环境变量，`yr_node_bootstrap.sh` 仅在值为 `true` 时追加： + +```text +--enable_sandbox_pause_resume true +--snapshot_storage_backend datasystem +--checkpoint_dir /home/akernel/sandboxd/root/checkpoints +``` + +首轮 standalone 不开放任意 backend 或任意 checkpoint path，以缩小配置面。无 RRT +能力的镜像收到 `AKERNEL_ENABLE_PAUSE_RESUME=true` 时应在启动早期给出明确错误，而不是 +启动后等待首个 pause 请求失败。 + +#### 共享 checkpoint root + +固定 checkpoint root： + +```text +/home/akernel/sandboxd/root/checkpoints +``` + +理由：standalone 把宿主机 `deploy/standalone/data/` 挂载到容器 +`/home/akernel`；sandboxd `rootDir` 是 `/home/akernel/sandboxd/root`，其 checkpoint +artifact manager 只接受该 root 下的路径。FunctionAgent、RuntimeManager 和 sandboxd +必须看到完全相同的绝对路径，不能使用 YuanRong process 默认的 +`/home/yuanrong/checkpoints`，也不能用 symlink 绕过路径所有权校验。 + +#### SnapshotStorage + +首轮使用同容器 DataSystem worker： + +```text +host = 127.0.0.1（或 process 脚本解析出的本节点 IP） +port = 31501（沿用实际 DS_WORKER_PORT） +backend = datasystem +``` + +不需要 OBS 凭据。Pause 时 FunctionAgent 读取本地 runsc checkpoint artifact，上传为以 +snapshot/request ID 标识的 DataSystem 对象；Resume 时下载到受控 attempt 目录，再由 +RuntimeManager 调用 sandboxd Restore。 + +### E2E 验收设计 + +#### 客户端环境 + +E2E runner 在 x86 Linux 宿主机创建独立 venv，下载并校验上表的 +`openyuanrong-sandbox` wheel 后安装。它使用 standalone 生成的服务器地址和 token， +不依赖本仓库的 `akernel_sdk` 实现。 + +#### 主流程 + +测试一次运行执行以下断言： + +1. 创建 `runtime="runsc"` 的 RRT sandbox，记录 sandbox ID，并确认 `get_info()` 为 + `running`。 +2. 写入随机 marker 文件并校验内容。 +3. 启动一个 `background=True, stdin=True` 的 shell 进程，让它阻塞等待输入；记录 PID。 +4. 调用 `pause(ttl_seconds=900)`： + - SDK 返回相同 sandbox ID； + - state 为 `paused`； + - snapshot ID 非空且等于 SDK 内部 lifecycle request ID； + - size、expiresAt 均为正值； + - watcher 查询最终为 `paused`。 +5. 调用 `resume()`： + - SDK 返回相同 sandbox ID； + - state 为 `running`； + - routeAddress、functionProxyId 非空； + - watcher 查询最终为 `running`。 +6. 向 pause 前创建的 PID 写入 stdin 并等待退出，断言输出正确。这是验证“同一进程树由 + runsc restore 恢复”的核心证据，不只验证重新创建了一个空 sandbox。 +7. 重新读取 marker 文件，再执行一条新命令，分别验证文件系统连续性和恢复后的 RRT + RuntimeRPC/直连数据面。 +8. 再执行一次 pause/resume，断言产生新的 snapshot ID，并重复最小健康检查。 +9. 删除 sandbox，确认实例消失；无论中间成功或失败都执行幂等清理。 + +#### 白盒证据和失败诊断 + +runner 输出一个不含 token 的 JSON 报告，至少包含： + +- 镜像引用、AKernel/sandboxd revision 与 process compatibility patch identity +- SDK、core、RRT artifact 版本和 SHA-256 +- sandbox ID、两次 snapshot ID、size、expiresAt +- pause/resume 开始与结束时间、返回 route/node/proxy 信息 +- continuity 断言结果 + +失败时收集但不提交： + +- `journalctl -u yuanrong.service -u sandboxd.service` +- FunctionProxy/FunctionAgent/RuntimeManager 日志 +- sandboxd 和 runsc 日志 +- DataSystem worker 日志 +- ETCD 中该 sandbox 的 lifecycle 状态 +- checkpoint root 的目录元数据（不复制 checkpoint 内容） + +snapshot ID 等于 pause request ID，可作为跨 Frontend、FunctionSystem、DataSystem 和本地 +artifact 的关联键。日志收集脚本必须对 token、IAM seed、registry auth 和云凭据做脱敏。 + +#### 失败与恢复语义 + +- wheel URL/SHA 不成对、checksum 不符、RRT 不是 x86_64 ELF、compatibility patch + 上下文不匹配时，镜像构建立即失败，不生成可部署 tag。 +- `AKERNEL_ENABLE_PAUSE_RESUME=true` 但镜像无 RRT capability 或共享 checkpoint root + 不可写时，standalone 在服务就绪前失败并打印明确原因。 +- SDK 对一次 lifecycle 调用的 transport retry 复用同一 request ID；Pause 返回的 + snapshot ID 必须与之相同，从而保证重试不会创建多个逻辑 snapshot。 +- Pause 未返回权威 `paused` 终态时，runner 不发 Resume，并以 sandbox ID/request ID + 收集 ETCD、FunctionSystem、DataSystem、sandboxd 与 runsc 证据。 +- Resume 未返回权威 `running` 路由时，runner 不创建替代 sandbox 掩盖失败；它保留现场 + 直到诊断收集完成，随后执行幂等 Delete。 +- 第二轮 pause/resume 或 Delete 失败均判定整次 E2E 失败，避免只验证 happy path 而忽略 + 生命周期 worker、checkpoint attempt 或 DataSystem 对象泄漏。 + +### 测试计划 + +- 单元测试：隔离验证 URL/SHA 成对规则、amd64 限制、RRT services 选择、feature 默认值、 + bootstrap 参数构造和日志脱敏。删除任一 gate/backend/root 参数都必须使测试失败。 +- 集成测试：构建 runtime stage 并检查 wheel SHA、ELF 架构、可执行位和 EROFS 目标路径； + 检查 node image 中 core/RRT revision、唯一 `rrt` slot、process wiring 以及 sandboxd RPC。 +- 端到端测试：在 privileged Linux x86_64 standalone 上执行“主流程”的两轮生命周期，重点 + 证明 pause 前 PID 和文件在 resume 后连续，而不是只断言 API 返回 200。 +- 故障与清理测试：验证 patch 上下文损坏时构建失败、feature 默认关闭、显式开启时真实 + 进程 argv 正确；teardown 后检查 sandbox、runtime、checkpoint attempt 和临时对象无 + 泄漏。DataSystem 或 runsc 的强制故障注入不进入首轮通过门槛，但必须保留诊断入口。 + +### 兼容性与已知限制 + +#### #215 不是当前 FunctionSystem 分支最新提交 + +#215 的 FunctionSystem 为 `6fac483a`。参考实现工作树在其后还有 snapshot worker +self-await 修复。首轮以 #215 进行 bring-up，若 E2E 在 pause/resume teardown 或重复操作 +暴露该问题，不单独替换 FunctionSystem 二进制；应从包含修复的同一 YuanRong 顶层提交 +重新运行整组 Buildkite artifact，保持 Frontend、FunctionSystem、RRT、Sandbox SDK +版本一致后重测。 + +#### arm64 macOS 只承担非执行型验证 + +当前本机是 Darwin arm64，而 #215 的 core/RRT 原生载荷是 Linux x86_64。本机只完成 +源码、脚本、wheel 内容、checksum、ELF header、patch context 和测试代码验证；不直接 +执行 `rrt-runtime`，也不把 `docker buildx --platform linux/amd64` 的 QEMU 模拟构建作为 +正式镜像证据。all-in-one 的 `make build`、镜像启动和 runsc checkpoint/restore 必须在 +Linux x86_64、privileged Docker、内核能力满足的服务器完成，并与本机验证分开报告。 + +#### compatibility patch 生命周期 + +兼容 patch 只服务于 #215 process-mode 缺口，并受 wheel checksum 和 patch context 双重 +约束。不能让它演变成长期 fork；上游产物包含等价配置后必须删除。 + +### 成熟度标准 + +只有同时满足以下条件才可声明端到端验证通过： + +- x86 all-in-one 镜像由 checksum-pinned core + RRT 构建成功； +- RRT service slot 实际可调度，且 pause feature 只在显式开关下开启； +- sandboxd Checkpoint/Restore、DataSystem upload/download、runsc restore 均有可关联证据； +- pause 前后台 PID 在 resume 后继续完成； +- 文件内容、同一 sandbox ID、恢复路由和新命令均正确； +- 第二轮 pause/resume 通过； +- 删除和 teardown 无资源泄漏或已知 self-await； +- 报告记录全部 artifact/revision，且不包含任何凭据。 + +### 升级与回滚策略 + +升级时先构建新 tag，保持旧镜像可用；以默认关闭状态启动并通过静态/健康检查后，再在 +专用 standalone 节点显式开启 feature。回滚前停止新的 Pause 请求，并确保已 pause 的 +sandbox 已成功 Resume 或 Delete；随后停止 standalone，使用原数据目录启动旧镜像。兼容 +patch 只存在于新镜像层，不修改宿主机配置或云资源。已创建但未清理的 checkpoint 和 +DataSystem 对象不能由旧镜像解释，必须在回滚前按新版本的清理路径处理。 + +## 生产就绪评审 + +- 特性开关和回滚：具备默认关闭的 standalone 开关；尚无按租户或请求粒度的 rollout。 +- 指标、日志和告警：首轮提供关联 ID 和诊断包；尚未定义长期 SLO、告警阈值和 dashboard。 +- 外部依赖：要求 Linux x86_64、privileged Docker、runsc checkpoint 能力和本地 + DataSystem worker。 +- 容量与扩展性：首轮仅验证单 sandbox、单节点；未验证并发 Pause、对象容量和 GC。 +- 运维工具：具备启动前检查、失败日志收集和幂等 Delete；不具备生产级 orphan repair。 + +因此该设计的成熟度上限是实验性 E2E 验证，不满足生产 GA 条件。 + +## 实施历史 + +- 2026-08-12：基于 AKernel builder、YuanRong 参考设计和 Buildkite #215 实物核验形成草案。 + +## 缺点 + +- 为复用 #215 引入临时 process compatibility patch，增加了一处需要上游收敛的维护点。 +- 单一 EROFS rootfs 同时承载四套 Python runtime 和 RRT，镜像尺寸不会因只使用 RRT 而 + 缩小。 +- 首轮 DataSystem backend 不能证明跨进程重启或跨节点持久化恢复能力。 + +## 备选方案 + +### 先补 YuanRong process 脚本并重新跑 Buildkite + +在 YuanRong 顶层和 FunctionSystem 仓补齐 process-mode 参数，再构建新的整组 wheel。 +该方案归属更干净且不需要 AKernel compatibility patch，但不能直接消费已经完成的 #215。 +它是兼容层的最终收敛方向，不作为首轮 bring-up 前置条件。 + +### 直接使用 YuanRong Kubernetes runtime 镜像 + +部署 #215 生成的 Kubernetes 镜像和 Helm chart 最接近 YuanRong 原生交付，但会绕过 +AKernel `builder/` 和 all-in-one 镜像，无法验证本任务关注的 x86 编译与 standalone +集成，因此不采用。 + +### 首轮改用 OBS SnapshotStorage + +OBS 可以验证进程重启后的持久对象，但会引入 endpoint、bucket、AK/SK、网络和清理策略， +扩大首轮故障面。DataSystem 已是真实 SnapshotStorage，足以验证当前单节点数据面；OBS +留到基础闭环稳定后单独设计。 + +## 基础设施需求 + +- 一台 Linux x86_64 服务器，Docker daemon 可用并允许 privileged container。 +- 服务器能下载 #215 OBS wheel、基础镜像、gVisor 和 Kata 构建依赖；私网地址应加入 + `NO_PROXY`，不把代理注入 sandbox 内部控制链路。 +- 足够存放两阶段 Docker build、224 MiB core wheel、EROFS rootfs、all-in-one image 和 + standalone 数据目录的本地磁盘空间。 +- 不需要 Kubernetes、OBS 写入凭据或新增云资源。 diff --git a/docs/superpowers/specs/2026-08-14-akernel-buildkite-image-pipeline-design.md b/docs/superpowers/specs/2026-08-14-akernel-buildkite-image-pipeline-design.md new file mode 100644 index 0000000..280e19e --- /dev/null +++ b/docs/superpowers/specs/2026-08-14-akernel-buildkite-image-pipeline-design.md @@ -0,0 +1,175 @@ +# AKernel Buildkite Image Pipeline Design + +## Goal + +Provide a repository-owned Buildkite pipeline that builds and pushes one +universal AKernel image, consumes YuanRong artifacts either from a public +release or from a selected YuanRong Buildkite build, and publishes the +YuanRong sandbox SDK plus standalone and Helm deployment bundles. + +## Decisions + +- Build one universal image. Standalone and Kubernetes are deployment modes, + not separate image products. +- Push images to + `swr.cn-southwest-2.myhuaweicloud.com/openyuanrong/akernel-all-in-one` by + default. +- Keep the pipeline definition and all orchestration scripts in the AKernel + repository. +- Resolve a YuanRong source once into a machine-readable artifact manifest. + Downstream jobs consume that manifest and do not contain source-specific + branches. +- Use immutable, generated image tags and record the registry digest in every + deployment artifact. +- Never store API or registry credentials in pipeline YAML, build metadata, or + uploaded artifacts. Buildkite and SWR credentials come from agent/Kubernetes + secrets. + +## Inputs + +| Variable | Meaning | Default | +| --- | --- | --- | +| `YR_SOURCE` | `release` or `buildkite` | `release` | +| `YR_VERSION` | YuanRong release version | `0.9.7` | +| `YR_PIPELINE` | Source Buildkite pipeline slug | `yuanrong-jcl` | +| `YR_BUILD_NUMBER` | Passed YuanRong build number | required for `buildkite` | +| `YR_BUILDKITE_ORG` | Source Buildkite organization | `openyuanrong` | +| `AKERNEL_DEPLOY_TARGETS` | `standalone`, `helm`, or comma-separated both | both | +| `AKERNEL_IMAGE_REPOSITORY` | Destination image repository | Guiyang SWR AKernel repository | +| `AKERNEL_IMAGE_TAG` | Explicit destination tag | generated | +| `AKERNEL_INCLUDE_KATA` | Include Kata runtime assets | `true` | +| `AKERNEL_INCLUDE_NVIDIA` | Include NVIDIA userspace assets | `true` | + +## Source Resolution + +### Release + +The resolver downloads the architecture-specific `openyuanrong_core` wheel +and raw `rrt-runtime-amd64` binary from the selected GitHub release, verifies +their published `.sha256` files, and downloads the same-version +`openyuanrong-sandbox` universal wheel using PyPI JSON metadata. The sandbox +wheel is not currently present in the YuanRong GitHub release, so PyPI is the +authoritative release source for that artifact. + +### Buildkite + +The resolver reads the selected build through the Buildkite REST API and +requires a terminal `passed` state. It selects exact URLs from build metadata: + +- `obs-urls.build-all-amd64` for `openyuanrong_core` +- `obs-urls.build-rrt-amd64` for `openyuanrong_rrt` +- `obs-urls.test-sandbox-sdk`, with `build-all-amd64` as a compatibility + fallback, for `openyuanrong_sandbox` + +All selected artifacts are downloaded once to prove accessibility and compute +SHA-256 values. The resolver rejects missing, duplicate, malformed, or +wrong-kind artifacts. + +Custom core wheels are not assumed to be one of the two historical +pause/resume process-script patch targets. The existing patch helper keeps +strict SHA matching for those legacy wheels, accepts an already-satisfied +process contract, and otherwise leaves an unknown package unchanged in its +default `auto` mode. Its `require` mode remains available when a caller must +enforce that legacy process-script contract. + +The job obtains a read-capable Buildkite API token from a secret environment +variable. It never passes the token as a build input or emits it in logs. + +## Normalized Manifest + +The resolver writes `artifacts/yuanrong/artifact-manifest.json` with this +contract: + +```json +{ + "schema_version": 1, + "source": { + "type": "buildkite", + "organization": "openyuanrong", + "pipeline": "yuanrong-jcl", + "build_number": 221, + "commit": "..." + }, + "core": { + "kind": "wheel", + "filename": "openyuanrong_core-....whl", + "url": "https://...", + "sha256": "..." + }, + "rrt": { + "kind": "wheel", + "filename": "openyuanrong_rrt-....whl", + "url": "https://...", + "sha256": "..." + }, + "sandbox_sdk": { + "kind": "wheel", + "filename": "openyuanrong_sandbox-....whl", + "url": "https://...", + "sha256": "..." + } +} +``` + +Release manifests use `rrt.kind = "runtime"`. The image build wrapper maps +that distinction to the existing RRT wheel override or the new raw-runtime +override without changing Dockerfile source selection semantics. + +## Pipeline + +The checked-in bootstrap step uploads a dynamically generated pipeline: + +1. **Resolve YuanRong** validates inputs, resolves the selected source, + uploads the normalized manifest, and uploads the sandbox SDK wheel. +2. **Build and push image** downloads the manifest, initializes only the + sandboxd and distill-fs submodules, starts Docker in a privileged Buildkite + Kubernetes job, builds the RRT AKernel image, logs into SWR using injected + credentials, pushes it, resolves the registry digest, and uploads + `image-manifest.json` plus dockerd/build logs. +3. **Package deployments** downloads both manifests and the sandbox SDK, + emits only the requested deployment bundles, writes SHA256SUMS, and uploads + the results as Buildkite artifacts. + +The default queue is the existing Linux amd64 Kubernetes queue. The image job +uses a bounded privileged pod with an ephemeral Docker graph volume. No +credentials are written into the repository or uploaded bundles. + +## Deployment Artifacts + +The standalone bundle contains the tracked standalone scripts/configuration, +the YuanRong sandbox SDK wheel, a generated `image.env`, and both manifests. + +The Helm bundle contains the tracked AKernel chart, a generated +`values.image.yaml` pinning the generated image tag, the YuanRong sandbox SDK +wheel, and both manifests. Both bundles record the remote image digest for +auditing even though the current chart renders repository and tag separately. + +The final Buildkite artifacts are: + +- `openyuanrong_sandbox-*.whl` +- `akernel-standalone-.tar.gz` when requested +- `akernel-helm-.tgz` when requested +- `artifact-manifest.json` +- `image-manifest.json` +- `SHA256SUMS` + +## Failure Semantics + +- A non-passed or inaccessible YuanRong Buildkite build fails before image + work begins. +- An artifact checksum or filename mismatch fails source resolution. +- Missing registry credentials fail before `docker push`. +- The image step succeeds only after the pushed digest can be read back from + the registry. +- Deployment packaging consumes the pushed image manifest, so it cannot + publish a bundle for an unconfirmed image. +- Build logs and non-secret manifests are uploaded on failure where possible. + +## Validation + +Automated tests execute the resolver against controlled HTTP fixtures, parse +the emitted dynamic pipeline as YAML, inspect real generated tar archives, and +verify Make/build-script argument propagation. The remote acceptance gate +creates the Buildkite pipeline and runs both a Release-source build and a +YuanRong-Buildkite-source build, verifying image push, digest resolution, and +the complete artifact set. diff --git a/docs/superpowers/specs/2026-08-14-buildkite-large-dependency-pvc-cache-design.md b/docs/superpowers/specs/2026-08-14-buildkite-large-dependency-pvc-cache-design.md new file mode 100644 index 0000000..c2263ac --- /dev/null +++ b/docs/superpowers/specs/2026-08-14-buildkite-large-dependency-pvc-cache-design.md @@ -0,0 +1,140 @@ +# Buildkite Large Dependency PVC Cache Design + +## Goal + +Avoid repeatedly downloading immutable build dependencies through restricted +public egress on every AKernel image build. The cache covers the Kata +Containers 4.0.0 amd64 static archive, gVisor `release-20260706.0` amd64 +`runsc`, and the OpenTelemetry Collector contrib 0.120.0 amd64 archive. The +mechanism remains narrow enough to extend to other immutable large artifacts +without turning the cache into a package mirror or a source of build truth. + +## Authority and scope + +The upstream URL and repository-pinned digest remain authoritative. The PVC +is only a performance cache. Every cache hit and every fresh download must +match the repository-pinned digest before the artifact is consumed. Kata and +OpenTelemetry use SHA-256; gVisor uses the SHA-512 published alongside the +official `runsc` binary. + +The exact defaults are: + +| Component | Artifact | Size | Repository-pinned digest | +| --- | --- | ---: | --- | +| Kata 4.0.0 | `kata-static-4.0.0-amd64.tar.zst` | 1,952,994,060 bytes | SHA-256 `2c3b9dfeba355582b40aee462b12916c9740654d0230f696adf719d67b063a8c` | +| gVisor `release-20260706.0` | `runsc` for `x86_64` | 130,918,823 bytes | SHA-512 `73938c145ebe554cf61a01da455688f4b732eebdf7b1b635bdef5b195868b363d8cb400e3d92ed1f377b78996805556c247a4849583910cb04e92b156053033e` | +| OpenTelemetry Collector contrib 0.120.0 | `otelcol-contrib_0.120.0_linux_amd64.tar.gz` | 80,901,637 bytes | SHA-256 `81bf885bc9a86705feb3c113c5a356571390e3601eb651ffcf2b3428f6571adb` | + +Docker base images, package-manager caches, source checkouts, and the Docker +data root remain outside this cache. In particular, `/var/lib/docker` stays on +the per-job `emptyDir`; sharing a Docker data root between independent +daemons is unsafe. + +## Storage and lifecycle + +The Buildkite Kubernetes namespace contains one named local PVC for immutable +download artifacts. The image build job mounts it read-write at +`/var/cache/akernel-downloads`; checkout, YuanRong resolution, and deployment +packaging jobs do not mount it. A 10 GiB or larger `ReadWriteOnce` claim is +sufficient for the three default artifacts and leaves room for later +checksum-pinned archives. The Guiyang Buildkite cluster has no default +StorageClass, and its +Everest `csi-local-topology` class cannot provision on the amd64 builders +because those nodes have no `persistent` local-volume pool. The infrastructure +manifest therefore defines a static local PV backed by +`/mnt/paas/build-cache/akernel-dependency-cache` on the selected amd64 builder, +plus a `kubernetes.io/no-provisioner` StorageClass using +`WaitForFirstConsumer`. Node affinity co-locates the image job with that path. +The PV and StorageClass use `Retain`, so claim deletion cannot delete cached +files from the host. + +Cache paths include every identity dimension. The digest length identifies the +algorithm while preserving the existing Kata cache path: + +```text +kata////kata-static--.tar.zst +gvisor////runsc +otelcol-contrib//-//otelcol-contrib___.tar.gz +``` + +Changing the component, version, architecture, or digest therefore produces a +new entry without mutating the old one. A version override must be paired with +its matching digest. Cache eviction may remove any entry at any time; the next +build recreates it from the upstream source. + +## Data flow + +Before the node Docker build, the build driver checks all enabled deterministic +cache paths: + +1. If the final file exists and its repository-pinned digest matches, report a + cache hit. +2. If an artifact is missing or invalid, download it through the existing + WireGuard/Squid environment into a build-unique temporary file on the same + PVC. +3. Verify the temporary file, then atomically rename it to the deterministic + final path. +4. Pass the cache directory to BuildKit as a named local build context. +5. The Kata stage and final node stage mount that named context read-only. + Each consumer copies its exact artifact, verifies the repository-pinned + digest again, and only then installs or extracts it. + +Concurrent cache misses may perform duplicate upstream downloads, but they +cannot expose partial content: temporary names are unique and only verified +files are atomically published. This avoids persistent lock files and stale +lock recovery. A later optimization may add advisory locking if simultaneous +misses become common. + +## Compatibility and fallback + +The repository build interface remains usable outside Buildkite. When +`AKERNEL_DEPENDENCY_CACHE_DIR` is unset, the build supplies an empty local +named context and all three Docker consumers download from their current +upstream URLs. Setting the variable opts a caller into the validated host-side +cache. + +`GVISOR_RELEASE` and `GVISOR_AMD64_SHA512` form one override pair. The +OpenTelemetry version and SHA-256 form another pair. A base URL or exact URL +may independently select a mirror only when it serves identical bytes. +Invalid digest lengths or partial version/digest overrides fail before Docker +starts. + +The build driver checks that the installed Docker/BuildKit supports named +build contexts before using a configured cache and fails with an explicit +message if it does not. It never silently copies a multi-gigabyte archive into +the ordinary repository build context. + +## Failure handling + +- A corrupt cache hit is rejected and replaced from upstream. +- An interrupted download leaves only a build-unique temporary file; it is + removed by the downloader's exit trap and is never selected as a hit. +- A checksum mismatch fails the build and does not publish the temporary + file. +- An unavailable PVC prevents the image build pod from starting, making the + infrastructure problem visible rather than falling back to an unexpectedly + slow download. +- If the PVC is mounted and upstream is unavailable, a valid existing cache + hit still permits the build; a miss fails normally. +- Every Docker consumer independently verifies its digest, so a cache mutation + between prefetch and consumption cannot enter the image unnoticed. + +## Verification + +Tests cover SHA-256 and SHA-512 validation, deterministic paths for all three +components, valid hits, corrupt hits, digest mismatch, interrupted-download +cleanup, atomic publication, the no-cache fallback, BuildKit context wiring, +and the Kubernetes PVC mount being limited to the image build job. + +Buildkite acceptance uses two builds or two image-build job attempts: + +1. The first run records cache fills for gVisor and OpenTelemetry (and a Kata + hit if the existing entry is retained), validates every artifact, and + completes the image and deployment artifacts. +2. The second run records hits for Kata, gVisor, and OpenTelemetry, performs no + network download for those artifacts, revalidates every digest inside + Docker, and completes with the same configured versions and digests. + +The existing restricted-egress checks remain in force: WireGuard routes only +`10.77.0.1/32`, Buildkite and Huawei Cloud endpoints remain in `NO_PROXY`, and +non-allowlisted public domains still receive HTTP 403 from Squid. diff --git a/docs/superpowers/specs/2026-08-14-openyuanrong-downloaders-design.md b/docs/superpowers/specs/2026-08-14-openyuanrong-downloaders-design.md new file mode 100644 index 0000000..e57f9fe --- /dev/null +++ b/docs/superpowers/specs/2026-08-14-openyuanrong-downloaders-design.md @@ -0,0 +1,107 @@ +# 可替换的 openYuanRong 制品下载器 + +| 字段 | 值 | +|---|---| +| 编号 | AKERNEL-BUILD-20260814 | +| 状态 | 可实施 | +| 作者 | Codex | +| SIG / 模块 | AKernel Builder | +| 评审人 | AKernel 维护者 | +| 批准人 | ChamberlainJI | +| 创建日期 | 2026-08-14 | + +## 摘要 + +将 `openyuanrong_core` wheel 与 RRT runtime 的来源选择、下载、校验和格式转换从 Dockerfile 内联命令迁移到两个路径稳定的下载脚本。公开仓库提供兼容现有 Release 和 URL/SHA 覆盖参数的默认实现；私有流水线可在构建前替换同路径脚本，从 OBS 等来源获取制品，而无需修改 Dockerfile、Makefile 或公共构建参数。 + +## 背景与动机 + +`builder/node.Dockerfile` 当前内联构造 Release core wheel URL，并在收到 URL/SHA 参数时切换到 daily wheel。`builder/runtime.Dockerfile` 同时承担 Release 裸 RRT binary 下载和 OBS RRT wheel 解包。两处 Dockerfile 因此了解制品仓库、命名、校验和打包格式，私有流水线也必须持续跟随这些实现细节。 + +### 目标 + +- Dockerfile 不再包含 openYuanRong Release 或 OBS 的下载分支；静态契约测试可判定这一点。 +- core 下载器在目标目录中产生且仅产生一个 wheel，Dockerfile 验证后安装。 +- RRT 下载器产生最终裸 runtime，Dockerfile 独立验证其为可执行 x86-64 ELF。 +- 默认下载器保持现有 Release 下载、固定校验和以及 URL/SHA 覆盖行为。 +- 流水线仅替换两个稳定路径下的脚本即可接管制品来源，并通过 Docker COPY 缓存边界使替换生效。 + +### 非目标 + +- 不改变 openYuanRong 版本、固定校验和、镜像布局或 runtime profile。 +- 不引入 BuildKit named context、远程脚本执行或新的凭据传递机制。 +- 不移除现有 `OPEN_YR_CORE_WHEEL_*`、`OPEN_YR_RRT_WHEEL_*` 兼容参数。 +- 不修改 pause/resume compatibility patch 的选择语义。 + +## 方案概述 + +公开构建继续调用 `make build`，默认脚本按当前 Release 规则下载。需要 OBS 制品的流水线在 `docker build` 前用自己的实现覆盖： + +```text +builder/downloaders/download-openyuanrong-core.sh +builder/downloaders/download-openyuanrong-rrt.sh +``` + +core 脚本接收目标目录，必须在目录顶层留下恰好一个 `.whl`。RRT 脚本接收目标文件路径，必须写入可供 Dockerfile 验证的裸 runtime。脚本是构建上下文中的源文件；覆盖脚本会改变 COPY 层摘要，不会命中旧下载层缓存。 + +### 风险与缓解措施 + +| 风险 | 缓解措施 | +|---|---| +| 私有脚本产生空目录或多个 wheel | Dockerfile 在安装前严格验证恰好一个普通 `.whl` 文件。 | +| 下载器绕过完整性校验 | 默认脚本保留固定 SHA-256；私有脚本被视为流水线构建代码，测试夹具要求其自行校验来源。 | +| RRT wheel 与裸 binary 格式混淆 | 下载器统一输出裸 runtime，Dockerfile 继续执行权限和 x86-64 ELF 检查。 | +| 脚本替换未使 Docker 缓存失效 | 每个 Dockerfile 在下载 RUN 前单独 COPY 对应脚本。 | + +## 详细设计 + +### core 下载器契约 + +接口为： + +```bash +download-openyuanrong-core.sh DEST_DIR +``` + +默认实现根据 `TARGETARCH` 或宿主架构选择 `x86_64`/`aarch64` wheel 与固定 Release SHA-256。`OPEN_YR_CORE_WHEEL_URL` 和 `OPEN_YR_CORE_WHEEL_SHA256` 必须同时为空或同时非空；非空时下载任意合法 `.whl` basename。所有下载使用现有 curl 重试策略，并在文件进入目标目录前完成 SHA-256 校验。 + +`node.Dockerfile` 创建空目标目录、执行脚本、通过 shell glob 验证恰好一个普通 `.whl`，然后保持现有 `pip --no-deps --target` 安装、`yr` 二进制检查和安装目录复制流程。 + +### RRT 下载器契约 + +接口为： + +```bash +download-openyuanrong-rrt.sh DEST_FILE +``` + +默认实现未收到 wheel override 时下载 Release 裸 binary 并校验固定 SHA-256；收到成对的 `OPEN_YR_RRT_WHEEL_URL/SHA256` 时下载 wheel、校验并提取 `openyuanrong_rrt/rrt-runtime` 到目标文件。临时文件位于独立临时目录并通过 trap 清理，失败时不保留半成品目标。 + +`runtime.Dockerfile` 执行脚本后设置可执行权限，并保持现有 x86-64 ELF 检查。下载器负责“来源和格式”，Dockerfile 负责“镜像需要什么物理结果”。 + +### 兼容性和失败语义 + +现有 Makefile 与 `deploy/scripts/build-image.sh` 参数保持不变。默认构建生成与当前实现等价的网络请求和制品内容。参数不成对、未知架构、checksum 不匹配、wheel 缺少 RRT member、core 输出数量不为一或 RRT 非 ELF 时，构建立即失败。 + +私有脚本可定义自己的环境变量，但不能要求公共 Dockerfile 识别这些变量；流水线负责通过已有 Docker build 环境将变量提供给替换脚本。 + +### 测试计划 + +- 下载器单元测试使用本地 `file://` Release/OBS 夹具，验证默认 core、core URL override、默认裸 RRT、RRT wheel override 和 checksum 失败，不依赖公网。 +- Dockerfile 契约测试验证两个下载器被 COPY 并执行，core 单 wheel 与 RRT ELF 防线存在，同时拒绝重新出现内联 openYuanRong curl 下载分支。 +- 既有构建参数测试继续验证 URL/SHA 透传，确保升级不破坏现有流水线。 +- Shell 语法检查覆盖新增脚本；最终以精简 Docker build 或等价 stage 构建验证容器内工具差异。 + +### 升级与回滚策略 + +升级不改变调用参数，默认构建无需调整。私有流水线可逐条覆盖脚本并独立验证。回滚只需恢复 Dockerfile 内联下载提交；制品版本和镜像运行时数据格式均未改变。 + +## 备选方案 + +- 单一 dispatcher 脚本：文件更少，但私有覆盖实现必须同时理解 core wheel 与 RRT 两种结果契约，扩大故障域。 +- BuildKit named context：可以从外部注入脚本，但会要求所有构建入口切换到 buildx 特有接口，不符合现有 `docker build` 兼容目标。 +- 仅保留 URL/SHA build args：现状已经支持，但流水线仍需跟随 Dockerfile 内部的 Release/OBS 格式分支，不能满足无侵入替换需求。 + +## 实施历史 + +- 2026-08-14：方案批准并进入实施。 diff --git a/sdk/python/README.md b/sdk/python/README.md index 3e0d969..52eee77 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -55,8 +55,12 @@ Address behavior is deterministic: - A host or IP without a port uses HTTPS/WSS on 443 for the frontend and HTTP on 80 for public sandbox port URLs. - `host:port` uses that port as a shared HTTPS/WSS endpoint. -- `AKERNEL_GATEWAY_ADDRESS` overrides the port-forwarding and exec gateway for - standalone or custom topologies. An override without a scheme uses HTTP/WS. +- `AKERNEL_GATEWAY_ADDRESS` overrides the PTY, file, and reverse-tunnel gateway + for standalone or custom topologies. An override without a scheme uses + HTTP/WS. +- `AKERNEL_SANDBOX_ROUTER_ADDRESS` independently overrides the direct + SandboxRouter endpoint used only by public sandbox port URLs. It falls back + to the legacy gateway behavior when omitted. The legacy actor backend is optional. Install and select it before importing `akernel_sdk`: @@ -313,9 +317,9 @@ with Sandbox(port_forwardings=[8080]) as sandbox: server.kill() ``` -`get_port_url()` rejects undeclared ports. Pass `internal=True` only when a -deployment operator explicitly wants the direct Traefik address instead of the -public gateway. +`get_port_url()` rejects undeclared ports. Deployments with separate frontend +and SandboxRouter listeners should set `AKERNEL_SANDBOX_ROUTER_ADDRESS`; this +does not change the endpoint used by PTY, file, or reverse-tunnel traffic. ## Reverse tunnels diff --git a/sdk/python/akernel_sdk/_addresses.py b/sdk/python/akernel_sdk/_addresses.py index 0c29139..50d3abe 100644 --- a/sdk/python/akernel_sdk/_addresses.py +++ b/sdk/python/akernel_sdk/_addresses.py @@ -21,9 +21,10 @@ * ``host:port``: shared-port mode. Frontend API, exec WebSocket, and public port-forward URLs all use the explicit port with TLS by default. -``AKERNEL_GATEWAY_ADDRESS`` remains an explicit override for standalone or -custom network topologies. When it is set without a scheme, it is treated as a -plain HTTP/WebSocket gateway. +``AKERNEL_GATEWAY_ADDRESS`` remains an explicit override for PTY, file, and +reverse-tunnel WebSocket traffic. ``AKERNEL_SANDBOX_ROUTER_ADDRESS`` can +independently select the direct SandboxRouter endpoint used only by public +port-forward URLs. Overrides without a scheme use plain HTTP/WebSocket. """ from __future__ import annotations @@ -108,6 +109,13 @@ def _gateway_override_raw() -> str: ) +def _sandbox_router_override_raw() -> str: + return ( + os.environ.get("AKERNEL_SANDBOX_ROUTER_ADDRESS", "").strip() + or os.environ.get("YR_SANDBOX_ROUTER_ADDRESS", "").strip() + ) + + def api_endpoint_from_env() -> Endpoint: """Return the frontend API endpoint derived from AKERNEL_SERVER_ADDRESS.""" return _parse_endpoint( @@ -118,7 +126,7 @@ def api_endpoint_from_env() -> Endpoint: def gateway_endpoint_from_env() -> Endpoint: - """Return the public port-forwarding gateway endpoint. + """Return the legacy shared gateway endpoint. An explicit gateway override is parsed as plain HTTP by default because standalone exposes Traefik's web entrypoint without TLS. Without an @@ -149,6 +157,23 @@ def gateway_endpoint_from_env() -> Endpoint: ) +def port_endpoint_from_env() -> Endpoint: + """Return the endpoint used only by public sandbox port URLs. + + A direct SandboxRouter override is independent from the gateway used by + PTY, file, and reverse-tunnel traffic. Existing deployments without the + new variable retain the legacy shared-gateway behavior. + """ + override = _sandbox_router_override_raw() + if override: + return _parse_endpoint( + override, + default_port=DEFAULT_PUBLIC_PORT, + default_scheme="http", + ) + return gateway_endpoint_from_env() + + def exec_endpoint_from_env() -> Endpoint: """Return the endpoint used by the exec WebSocket (/terminal/ws). diff --git a/sdk/python/akernel_sdk/sandbox.py b/sdk/python/akernel_sdk/sandbox.py index 90f95e0..f88f4da 100644 --- a/sdk/python/akernel_sdk/sandbox.py +++ b/sdk/python/akernel_sdk/sandbox.py @@ -24,7 +24,7 @@ from types import MappingProxyType from typing import Literal, cast -from ._addresses import Endpoint, api_endpoint_from_env, gateway_endpoint_from_env +from ._addresses import Endpoint, api_endpoint_from_env, port_endpoint_from_env from ._backends.base import BackendSession, SandboxSpec from ._backends.registry import load_backend from ._sandbox_resources import normalize_xpu, validate_storage_mb @@ -352,7 +352,7 @@ def get_port_url(self, port: int, *, internal: bool = False) -> str: f"{sorted(self._forwarded_ports)}" ) - gateway = gateway_endpoint_from_env() + gateway = port_endpoint_from_env() if internal: pod_ip, gateway_port = _get_traefik_internal_ip(gateway) direct = Endpoint( diff --git a/sdk/python/tests/unit/test_addresses.py b/sdk/python/tests/unit/test_addresses.py index f3383b9..9d58d00 100644 --- a/sdk/python/tests/unit/test_addresses.py +++ b/sdk/python/tests/unit/test_addresses.py @@ -20,6 +20,7 @@ api_endpoint_from_env, exec_endpoint_from_env, gateway_endpoint_from_env, + port_endpoint_from_env, ) @@ -79,6 +80,25 @@ def test_gateway_override_respects_scheme(self): ("https", "gw.example.com", 9443, True), ) + def test_sandbox_router_override_only_changes_public_port_endpoint(self): + with patch.dict( + os.environ, + { + "AKERNEL_SERVER_ADDRESS": "https://frontend.example:8888", + "AKERNEL_GATEWAY_ADDRESS": "https://frontend.example:8888", + "AKERNEL_SANDBOX_ROUTER_ADDRESS": "http://router.example:8080", + }, + clear=True, + ): + self.assertEqual( + endpoint_tuple(port_endpoint_from_env()), + ("http", "router.example", 8080, False), + ) + self.assertEqual( + endpoint_tuple(exec_endpoint_from_env()), + ("https", "frontend.example", 8888, True), + ) + def test_internal_yr_gateway_does_not_override_exec_endpoint(self): with patch.dict( os.environ, diff --git a/sdk/python/tests/unit/test_sandbox.py b/sdk/python/tests/unit/test_sandbox.py index 9dfcf23..6a5b7c0 100644 --- a/sdk/python/tests/unit/test_sandbox.py +++ b/sdk/python/tests/unit/test_sandbox.py @@ -325,6 +325,23 @@ def test_get_port_url(self): sandbox.get_port_url(9090) sandbox.kill() + def test_get_port_url_prefers_sandbox_router_address(self): + with patch.dict( + os.environ, + { + "AKERNEL_SERVER_ADDRESS": "https://frontend.example.com:8888", + "AKERNEL_GATEWAY_ADDRESS": "https://frontend.example.com:8888", + "AKERNEL_SANDBOX_ROUTER_ADDRESS": "http://router.example.com:8080", + }, + clear=True, + ): + sandbox = Sandbox(port_forwardings=[8080]) + self.assertEqual( + sandbox.get_port_url(8080), + "http://router.example.com:8080/physical-id/8080", + ) + sandbox.kill() + if __name__ == "__main__": unittest.main() diff --git a/src/sandboxd b/src/sandboxd index 17b7846..55971e1 160000 --- a/src/sandboxd +++ b/src/sandboxd @@ -1 +1 @@ -Subproject commit 17b78468a5454ecb86b15ca0bc4f3461cb95f06a +Subproject commit 55971e14f533a79ac84d9e23ad455ca1e9d56c47