From dbf65a434aea87cf2aabb37e89fcafbcfa0876b3 Mon Sep 17 00:00:00 2001 From: Samuel Laferriere <9342524+samlaf@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:09:03 -0400 Subject: [PATCH 1/3] test(tee-cli): split the live cross-repo drift guards into `make test-live` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `make check` ran two network-dependent tests that skipped dynamically on transport failures — invisible in CI's single step, and one of them (the registry runtime-code pin check) has been silently skipping in CI forever because the admission binary is never installed there. Split them out so the default suite is hermetic and the live guards get their own attributable CI step. - `live_test_manifest.py` (new): the enclave manifest-fixture byte-parity test and the registry runtime-code pin test move here, sharing a `_fetch_live` helper. Failure taxonomy unchanged — HTTP 4xx/5xx (artifact moved / ref gone) is a real drift signal and fails — but transport errors (unreachable, timeout, truncated body) now retry once before skipping, and the skip reason prints in the target's verbose output. - The module name deliberately misses `make test`'s `test*.py` discovery pattern, so `make check` is now offline-deterministic: no network, no dynamic skips. - `make test-live` runs `unittest discover -p "live_test_*.py" -b -v`; CI runs it as its own step after `make check`, so a cross-repo drift failure is attributable from the step name alone. Known gap, now visible instead of silent: the runtime-pin test still skips in CI ("seismic-measurement-admission not in PATH") until CI installs the admission binary — e.g. a prebuilt artifact, same pattern as proposed for verify-quote's container image. --- .github/workflows/ci.yml | 5 ++ Makefile | 13 ++- tee/cli/common/tests/live_test_manifest.py | 99 ++++++++++++++++++++++ tee/cli/common/tests/test_manifest.py | 71 ++-------------- 4 files changed, 120 insertions(+), 68 deletions(-) create mode 100644 tee/cli/common/tests/live_test_manifest.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37b5e3d6..7559ead0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,3 +20,8 @@ jobs: - name: make check run: make check + + # Separate step so a cross-repo drift failure is attributable at a + # glance, and `make check` stays hermetic (offline, no dynamic skips). + - name: make test-live + run: make test-live diff --git a/Makefile b/Makefile index 95c600c6..14d4b523 100644 --- a/Makefile +++ b/Makefile @@ -2,16 +2,17 @@ PKG := deploy_gcp tee -.PHONY: help lint format format-check typecheck test check +.PHONY: help lint format format-check typecheck test test-live check help: @echo "Targets:" - @echo " check lint + format-check + typecheck + test (what CI runs)" + @echo " check lint + format-check + typecheck + test (hermetic; CI runs this)" @echo " lint ruff check" @echo " format ruff format (writes changes)" @echo " format-check ruff format --check" @echo " typecheck ty check" - @echo " test unittest discovery (repo-wide)" + @echo " test unittest discovery (repo-wide, offline)" + @echo " test-live cross-repo drift guards (network-required; own CI step)" check: lint format-check typecheck test @@ -31,3 +32,9 @@ typecheck: # passing runs stay quiet (the code under test prints progress + warnings). test: uv run python -m unittest discover -b + +# Live cross-repo drift guards: modules named live_test_*.py fall outside +# `test`'s default `test*.py` discovery, keeping it hermetic. Verbose, so a +# skip (binary not on PATH, network flake after retry) is visible per test. +test-live: + uv run python -m unittest discover -b -v -p "live_test_*.py" diff --git a/tee/cli/common/tests/live_test_manifest.py b/tee/cli/common/tests/live_test_manifest.py new file mode 100644 index 00000000..7fbaca6a --- /dev/null +++ b/tee/cli/common/tests/live_test_manifest.py @@ -0,0 +1,99 @@ +"""Live cross-repo drift guards (network-required; run via `make test-live`). + +These tests fetch pinned artifacts from sibling repos on GitHub and compare +them against what this repo renders or pins. The module name deliberately +does not match `make test`'s `test*.py` discovery pattern, so the default +suite stays hermetic (offline, deterministic); CI runs this module as its +own step, where a failure or skip names the exact cross-repo check. + +Run with: + make test-live +""" + +import http.client +import json +import unittest +import urllib.error +import urllib.request + +from eth_utils import keccak + +from tee.cli.common.manifest import compile_measurement_policy, render_manifest +from tee.cli.common.tests.test_manifest import ( + ADMISSION_BIN, + FIXTURE_MANIFEST, + promoted_policy_bytes, +) + + +def _fetch_live(url: str) -> bytes: + """Fetch a cross-repo artifact, or skip the calling test. + + An HTTP 4xx/5xx means the artifact moved or the ref is gone — a real + drift signal, not flaky network — so it propagates as a failure. A + transport-level failure (unreachable, timeout, truncated body) is + retried once, then skips: no verdict beats a false one, and the skip + reason is visible in this module's verbose run. + """ + last: Exception | None = None + for _ in range(2): + try: + with urllib.request.urlopen(url, timeout=10) as resp: + return resp.read() + except urllib.error.HTTPError: + raise + except (urllib.error.URLError, TimeoutError, http.client.HTTPException) as e: + last = e + raise unittest.SkipTest(f"cross-repo artifact unreachable: {last}") + + +class ManifestFixtureParityTests(unittest.TestCase): + """Byte-parity with the node-side manifest parser. + + The enclave repo pins the manifest fixture's exact bytes; deploy's + emitter must render the same dict to the same bytes. Fetched from + GitHub (the `seismic` branch) rather than assuming a sibling checkout + on disk, so the check runs in CI too. The fixture's network_id is also + pinned offline by test_manifest's + test_render_matches_enclave_network_id_vector; this adds the live + byte-level drift guard on top. + """ + + ENCLAVE_FIXTURE_URL = ( + "https://raw.githubusercontent.com/SeismicSystems/enclave/seismic/" + "crates/network-manifest/fixtures/network-manifest-v1.json" + ) + + def test_render_matches_enclave_fixture_bytes(self): + fixture = _fetch_live(self.ENCLAVE_FIXTURE_URL) + self.assertEqual(render_manifest(FIXTURE_MANIFEST), fixture) + + +class RuntimeCodeDriftTests(unittest.TestCase): + """Cross-repo drift guard for the registry runtime-code pin. + + The admission CLI pins keccak256 of the canonical MeasurementRegistry + deployed bytecode; the gates enforce that pin against the genesis alloc, + so a stale pin already fails assembly loudly. This test is the early + warning: the pin reported by the binary on PATH must match the artifact + the reth genesis builder installs. + """ + + REGISTRY_ARTIFACT_URL = ( + "https://raw.githubusercontent.com/SeismicSystems/seismic/main/" + "contracts/artifacts/MeasurementRegistry.json" + ) + + @unittest.skipUnless(ADMISSION_BIN, "seismic-measurement-admission not in PATH") + def test_admission_crate_pins_current_registry_runtime(self): + report = compile_measurement_policy(promoted_policy_bytes()) + artifact = json.loads(_fetch_live(self.REGISTRY_ARTIFACT_URL)) + runtime = artifact["deployedBytecode"]["object"].removeprefix("0x") + self.assertEqual( + report["registry_runtime_code_hash"], + "0x" + keccak(bytes.fromhex(runtime)).hex(), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tee/cli/common/tests/test_manifest.py b/tee/cli/common/tests/test_manifest.py index 3fb78c64..accaea38 100644 --- a/tee/cli/common/tests/test_manifest.py +++ b/tee/cli/common/tests/test_manifest.py @@ -5,14 +5,11 @@ """ import hashlib -import http.client import json import shutil import tempfile import tomllib import unittest -import urllib.error -import urllib.request from pathlib import Path from typing import Any @@ -104,21 +101,9 @@ def promoted_policy_bytes(measurement_id: str = "img.vhd") -> bytes: "0x8ef142e3f2bf15f8b201c4d8cda7848a9e846222c62b5615d4d36c7fccd98a24" ) -# The node-side parser pins these exact bytes in the enclave repo. Fetch its -# fixture from GitHub (the `seismic` branch) rather than assuming a sibling -# checkout on disk, so the cross-repo byte-parity check runs in CI too. The -# network_id value is also pinned offline by -# test_render_matches_enclave_network_id_vector, so this only adds a live drift -# guard; it skips when GitHub is unreachable. -ENCLAVE_FIXTURE_URL = ( - "https://raw.githubusercontent.com/SeismicSystems/enclave/seismic/" - "crates/network-manifest/fixtures/network-manifest-v1.json" -) - - -def _fetch_enclave_fixture() -> bytes: - with urllib.request.urlopen(ENCLAVE_FIXTURE_URL, timeout=10) as resp: - return resp.read() +# The node-side parser pins the fixture's exact bytes in the enclave repo; +# the live byte-parity check against it lives in live_test_manifest.py +# (network-required, `make test-live`). class RenderTests(unittest.TestCase): @@ -126,17 +111,6 @@ def test_render_matches_enclave_network_id_vector(self): rendered = render_manifest(FIXTURE_MANIFEST) self.assertEqual(compute_network_id(rendered), FIXTURE_NETWORK_ID) - def test_render_matches_enclave_fixture_bytes(self): - try: - fixture = _fetch_enclave_fixture() - except urllib.error.HTTPError: - # A 4xx/5xx means the fixture moved or the ref is gone — a real - # drift signal, not flaky network, so fail loudly. - raise - except (urllib.error.URLError, TimeoutError) as e: - self.skipTest(f"enclave fixture unreachable: {e}") - self.assertEqual(render_manifest(FIXTURE_MANIFEST), fixture) - def test_render_is_deterministic_under_key_order(self): shuffled = dict(reversed(list(FIXTURE_MANIFEST.items()))) self.assertEqual(render_manifest(shuffled), render_manifest(FIXTURE_MANIFEST)) @@ -438,42 +412,9 @@ def test_report_without_storage(self): inject_registry_genesis_storage(genesis, self.REGISTRY, report) -class RuntimeCodeDriftTests(unittest.TestCase): - """Cross-repo drift guard for the registry runtime-code pin. - - The admission CLI pins keccak256 of the canonical MeasurementRegistry - deployed bytecode; the gates enforce that pin against the genesis alloc, - so a stale pin already fails assembly loudly. This test is the early - warning: the pin reported by the binary on PATH must match the artifact - the reth genesis builder installs. Online-only, like the - manifest-fixture byte-parity test. - """ - - REGISTRY_ARTIFACT_URL = ( - "https://raw.githubusercontent.com/SeismicSystems/seismic/main/" - "contracts/artifacts/MeasurementRegistry.json" - ) - - def _fetch(self, url: str) -> bytes: - try: - with urllib.request.urlopen(url, timeout=10) as resp: - return resp.read() - except urllib.error.HTTPError: - # A 4xx/5xx means the artifact moved — a real drift signal, - # not flaky network, so fail loudly. - raise - except (urllib.error.URLError, TimeoutError, http.client.HTTPException) as e: - self.skipTest(f"cross-repo artifact unreachable: {e}") - - @unittest.skipUnless(ADMISSION_BIN, "seismic-measurement-admission not in PATH") - def test_admission_crate_pins_current_registry_runtime(self): - report = compile_measurement_policy(promoted_policy_bytes()) - artifact = json.loads(self._fetch(self.REGISTRY_ARTIFACT_URL)) - runtime = artifact["deployedBytecode"]["object"].removeprefix("0x") - self.assertEqual( - report["registry_runtime_code_hash"], - "0x" + keccak(bytes.fromhex(runtime)).hex(), - ) +# The registry runtime-code drift guard (admission binary's pin vs the +# monorepo's MeasurementRegistry artifact) lives in live_test_manifest.py +# (network-required, `make test-live`). class GateTests(unittest.TestCase): From fd9a9a9e9be7fce44a46c527c148c46795bd9be4 Mon Sep 17 00:00:00 2001 From: Samuel Laferriere <9342524+samlaf@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:13:45 -0400 Subject: [PATCH 2/3] refactor ci.yml --- .github/workflows/ci.yml | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7559ead0..8973b1e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: push: - branches: [ main ] + branches: [main] pull_request: jobs: @@ -10,18 +10,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v4 + - run: uv sync --all-extras --dev - - name: Install uv - uses: astral-sh/setup-uv@v4 + - run: make check - # Also installs the Python version pinned in .python-version. - - name: Install dependencies - run: uv sync --all-extras --dev - - - name: make check - run: make check + live-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v4 + - run: uv sync --all-extras --dev - # Separate step so a cross-repo drift failure is attributable at a - # glance, and `make check` stays hermetic (offline, no dynamic skips). - - name: make test-live - run: make test-live + - run: make test-live From 0ae6fdc94f70dbc69937028b79b037686f5b0d06 Mon Sep 17 00:00:00 2001 From: Samuel Laferriere <9342524+samlaf@users.noreply.github.com> Date: Wed, 5 Aug 2026 17:01:28 -0400 Subject: [PATCH 3/3] =?UTF-8?q?ci:=20the=20live=20drift=20guards=20never?= =?UTF-8?q?=20skip=20=E2=80=94=20fail=20loudly,=20install=20their=20binary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The registry runtime-code pin test has skipped in CI forever (`seismic-measurement-admission` is never on the runner's PATH), and both live tests skipped on network flakes — so the live-tests job could go green while verifying nothing. Two changes make its green mean something: - The live suite never skips. It exists only to be run deliberately, and "ran the drift guards" must mean "got verdicts": a missing admission binary fails with a pointer to the enclave repo, and an unreachable artifact fails after one retry instead of skipping. (`make check`'s binary-gated subprocess tests keep their graceful skip — the default suite degrading politely on machines without the binary is a different, correct contract.) - CI's live-tests job installs the binary: `cargo install --git` from the public enclave repo (small crate — alloy-primitives + clap, none of the enclave workspace's TPM deps), with an actions/cache keyed on enclave's seismic-branch HEAD so warm runs skip the build. Building against enclave's moving branch is the point — the guard asks "does enclave HEAD's pin still match seismic main's artifact" — and also why the install stays out of the required `check` job: cross-repo drift must not redden unrelated PRs. --- .github/workflows/ci.yml | 25 ++++++++++++++++++++ Makefile | 3 +-- tee/cli/common/tests/live_test_manifest.py | 27 ++++++++++++++++------ 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8973b1e7..8fb1dbb8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,4 +22,29 @@ jobs: - uses: astral-sh/setup-uv@v4 - run: uv sync --all-extras --dev + # Without the binary the registry runtime-code pin test skips, and a + # drift-guard job that always skips is dead weight. Built from enclave's + # moving seismic branch — which is the point (guard against enclave + # HEAD), and why it stays out of the required `check` job: the crate is + # small (none of the enclave workspace's TPM deps), and the cache + # re-keys only when enclave moves. + - name: Resolve enclave HEAD + id: enclave + run: >- + echo "rev=$(git ls-remote https://github.com/SeismicSystems/enclave.git + refs/heads/seismic | cut -f1)" >> "$GITHUB_OUTPUT" + - name: Cache seismic-measurement-admission + id: admission-cache + uses: actions/cache@v4 + with: + path: ~/.cargo/bin/seismic-measurement-admission + key: seismic-measurement-admission-${{ steps.enclave.outputs.rev }} + - name: Build seismic-measurement-admission + if: steps.admission-cache.outputs.cache-hit != 'true' + run: >- + cargo install --locked + --git https://github.com/SeismicSystems/enclave + --rev ${{ steps.enclave.outputs.rev }} + seismic-measurement-admission --features cli + - run: make test-live diff --git a/Makefile b/Makefile index 14d4b523..def01e17 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,6 @@ test: uv run python -m unittest discover -b # Live cross-repo drift guards: modules named live_test_*.py fall outside -# `test`'s default `test*.py` discovery, keeping it hermetic. Verbose, so a -# skip (binary not on PATH, network flake after retry) is visible per test. +# `test`'s default `test*.py` discovery, keeping it hermetic. test-live: uv run python -m unittest discover -b -v -p "live_test_*.py" diff --git a/tee/cli/common/tests/live_test_manifest.py b/tee/cli/common/tests/live_test_manifest.py index 7fbaca6a..1d25d983 100644 --- a/tee/cli/common/tests/live_test_manifest.py +++ b/tee/cli/common/tests/live_test_manifest.py @@ -4,7 +4,15 @@ them against what this repo renders or pins. The module name deliberately does not match `make test`'s `test*.py` discovery pattern, so the default suite stays hermetic (offline, deterministic); CI runs this module as its -own step, where a failure or skip names the exact cross-repo check. +own non-required job, where a failure names the exact cross-repo check. + +The suite never skips — a missing prerequisite is a failure. It needs: + +- network reach to raw.githubusercontent.com (both tests fetch pinned + artifacts from sibling repos); +- `seismic-measurement-admission` on PATH — the enclave repo's admission + CLI (`cargo install --features cli` from crates/measurement-admission; + CI builds it from enclave's seismic branch). Run with: make test-live @@ -27,13 +35,12 @@ def _fetch_live(url: str) -> bytes: - """Fetch a cross-repo artifact, or skip the calling test. + """Fetch a cross-repo artifact, failing the calling test if it can't. An HTTP 4xx/5xx means the artifact moved or the ref is gone — a real - drift signal, not flaky network — so it propagates as a failure. A + drift signal, not flaky network — so it propagates directly. A transport-level failure (unreachable, timeout, truncated body) is - retried once, then skips: no verdict beats a false one, and the skip - reason is visible in this module's verbose run. + retried once, then fails: this suite never skips. """ last: Exception | None = None for _ in range(2): @@ -44,7 +51,9 @@ def _fetch_live(url: str) -> bytes: raise except (urllib.error.URLError, TimeoutError, http.client.HTTPException) as e: last = e - raise unittest.SkipTest(f"cross-repo artifact unreachable: {last}") + raise AssertionError( + f"cross-repo artifact unreachable after retry: {last}" + ) from last class ManifestFixtureParityTests(unittest.TestCase): @@ -84,8 +93,12 @@ class RuntimeCodeDriftTests(unittest.TestCase): "contracts/artifacts/MeasurementRegistry.json" ) - @unittest.skipUnless(ADMISSION_BIN, "seismic-measurement-admission not in PATH") def test_admission_crate_pins_current_registry_runtime(self): + self.assertIsNotNone( + ADMISSION_BIN, + "seismic-measurement-admission not on PATH — the live suite fails " + "rather than skips; build the enclave repo's admission CLI", + ) report = compile_measurement_policy(promoted_policy_bytes()) artifact = json.loads(_fetch_live(self.REGISTRY_ARTIFACT_URL)) runtime = artifact["deployedBytecode"]["object"].removeprefix("0x")