From 7c5731209020806f388deeec1e8872ef3d7d5724 Mon Sep 17 00:00:00 2001 From: plancher Date: Sun, 5 Jul 2026 13:21:21 -0400 Subject: [PATCH] expected-skips baseline for verify; derive signer from gh CLI keyholder --- README.md | 5 +- docs/local_mode.md | 25 ++++++++++ src/pytest_gpu_proof/cli.py | 11 +++++ src/pytest_gpu_proof/gitutils.py | 21 +++++++++ src/pytest_gpu_proof/receipt.py | 22 +++++++-- src/pytest_gpu_proof/verify.py | 62 +++++++++++++++++++++++-- tests/conftest.py | 12 +++++ tests/test_receipt.py | 44 ++++++++++++++++++ tests/test_verifier.py | 80 ++++++++++++++++++++++++++++++++ 9 files changed, 271 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 72332fa..8d3b973 100644 --- a/README.md +++ b/README.md @@ -225,7 +225,7 @@ python -m pytest_gpu_proof verify --receipt gpu-proof.json 1. **Signature** — fetches `github.com/{signer}.keys`, verifies Ed25519/ECDSA/RSA signature 2. **Fingerprint** — recomputes SHA-256 digest of `src/` and `tests/`, compares to receipt 3. **Commit SHA** — compares receipt commit SHA to current HEAD -4. **Test outcomes** — all tests recorded in the receipt must have passed; skipped marked tests fail verification unless `--allow-skipped` is passed +4. **Test outcomes** — all tests recorded in the receipt must have passed; skipped marked tests fail verification unless `--allow-skipped` is passed, or an `--expected-skips` baseline is given and the receipt's skip set matches it exactly 5. **Freshness** — receipt must be younger than `max_age_days` (default: 30) 6. **Dirty policy** — configurable via policy file 7. **GPU info** (optional) — with `--require-gpu` (or `require_gpu = true` in `[tool.gpu_proof]`), the receipt's `environment.gpu_info` must be present @@ -233,7 +233,8 @@ python -m pytest_gpu_proof verify --receipt gpu-proof.json Additional flags: - `--allow-unsigned` — accept receipts with `"signature": null` (produced by `--gpu-proof-signing-backend=none`). This disables the entire trust story; the verifier prints a loud warning. -- `--allow-skipped` — accept receipts that contain skipped marked tests. +- `--allow-skipped` — accept receipts that contain skipped marked tests (any skips, no questions asked). +- `--expected-skips PATH` — the strict alternative to `--allow-skipped` (the two are mutually exclusive): a baseline file of node IDs (one per line, `#` comments) that the receipt's skipped tests must match **exactly**. A skip not in the baseline fails verification, and a baseline entry that no longer skips fails too (stale baseline — update it). Use this when a project has a small set of permanent, documented skips. Can also be set as an inline list via `expected_skips = [...]` in `[tool.gpu_proof]`. - `--require-gpu` — reject receipts whose `environment.gpu_info` is null/absent. This is **modest hardening, not proof**: `gpu_info` is self-reported by the recording machine, so it only guards against accidentally signing on a GPU-less box, not against a dishonest signer. --- diff --git a/docs/local_mode.md b/docs/local_mode.md index 31cf930..e5895d9 100644 --- a/docs/local_mode.md +++ b/docs/local_mode.md @@ -72,6 +72,31 @@ The receipt is a human-readable JSON file. It is safe to commit — it contains No GPU, no secrets, no CUDA dependencies required in CI. +If your suite has a small set of permanent, documented skips, pin them instead +of waving all skips through: keep a baseline file (one node ID per line, `#` +comments allowed) and verify with + +```yaml +- name: Verify GPU proof receipt + run: gpu-proof verify --receipt gpu-proof.json --expected-skips expected_skips.txt +``` + +Verification then fails on any skip *not* in the baseline (something new is +being silently skipped) **and** on any baseline entry that no longer skips +(the baseline is stale — update it). `--allow-skipped` remains the loose +alternative and the two are mutually exclusive. + +## Who signs the receipt + +The signer identity recorded in the receipt (and whose `github.com/.keys` +CI verifies against) is resolved in order: +1. `--gpu-proof-github-user` / `github_username` in `[tool.gpu_proof]` +2. the authenticated GitHub CLI login (`gh api user`), if `gh` is available — + this is the actual keyholder +3. the origin remote owner, as a last-resort guess — with a warning, because + for org-owned repos this is the **org**, which has no SSH keys, and + verification would fail. Set option 1 or 2 up properly in that case. + ## Controlling which key is used By default the plugin tries these in order: diff --git a/src/pytest_gpu_proof/cli.py b/src/pytest_gpu_proof/cli.py index 3200a4a..1d09645 100644 --- a/src/pytest_gpu_proof/cli.py +++ b/src/pytest_gpu_proof/cli.py @@ -53,6 +53,16 @@ def main(): default=False, help="Accept receipts that contain skipped marked tests (default: reject)", ) + vp.add_argument( + "--expected-skips", + default=None, + metavar="PATH", + help="Baseline file of node IDs (one per line, '#' comments) that the " + "receipt's skipped tests must match EXACTLY — new skips fail, and a " + "baselined test that now runs flags the baseline as stale. Stricter " + "than (and mutually exclusive with) --allow-skipped. Can also be set " + "as an inline list via expected_skips in [tool.gpu_proof].", + ) vp.add_argument( "--require-gpu", action="store_true", @@ -76,6 +86,7 @@ def main(): allow_unsigned=args.allow_unsigned, allow_skipped=args.allow_skipped, require_gpu=args.require_gpu, + expected_skips_path=args.expected_skips, ) sys.exit(0 if ok else 1) diff --git a/src/pytest_gpu_proof/gitutils.py b/src/pytest_gpu_proof/gitutils.py index c4025dc..ee290b9 100644 --- a/src/pytest_gpu_proof/gitutils.py +++ b/src/pytest_gpu_proof/gitutils.py @@ -51,6 +51,27 @@ def get_github_username() -> Optional[str]: return extract_github_username(url) if url else None +def get_gh_cli_login() -> Optional[str]: + """The authenticated GitHub CLI user, if `gh` is installed and logged in. + + This is the KEYHOLDER — the account whose github.com/.keys will + verify the receipt — unlike the origin-remote owner, which for org-owned + repos is an org with no SSH keys. + """ + try: + result = subprocess.run( + ["gh", "api", "user", "--jq", ".login"], + capture_output=True, + text=True, + check=True, + timeout=10, + ) + return result.stdout.strip() or None + except (subprocess.CalledProcessError, FileNotFoundError, + subprocess.TimeoutExpired): + return None + + def get_git_signing_key() -> Optional[str]: val = _git("config", "--get", "user.signingKey") if not val: diff --git a/src/pytest_gpu_proof/receipt.py b/src/pytest_gpu_proof/receipt.py index f048f43..910141e 100644 --- a/src/pytest_gpu_proof/receipt.py +++ b/src/pytest_gpu_proof/receipt.py @@ -16,6 +16,7 @@ from .gitutils import ( get_branch, get_commit_sha, + get_gh_cli_login, get_github_username, get_remote_url, is_dirty, @@ -82,11 +83,22 @@ def build_receipt_payload( from .fingerprint import compute_fingerprint remote_url = get_remote_url() - github_username = ( - override_github_username - or config.github_username - or get_github_username() - ) + # Signer resolution: explicit override > [tool.gpu_proof]/flag config > + # authenticated gh CLI login (the actual keyholder) > origin-remote owner. + # The last is only a guess — for org-owned repos it yields the ORG, which + # has no SSH keys, so verification would fail; warn when we land there. + github_username = override_github_username or config.github_username + if not github_username: + github_username = get_gh_cli_login() + if not github_username: + github_username = get_github_username() + if github_username: + print( + f"[gpu-proof] WARNING: signer '{github_username}' was derived " + "from the origin remote owner, which may be an org with no SSH " + "keys. If verification fails, set --gpu-proof-github-user or " + "github_username in [tool.gpu_proof] to the keyholder." + ) fingerprint = compute_fingerprint(config.fingerprint_paths) return { diff --git a/src/pytest_gpu_proof/verify.py b/src/pytest_gpu_proof/verify.py index ddeb98a..72e097f 100644 --- a/src/pytest_gpu_proof/verify.py +++ b/src/pytest_gpu_proof/verify.py @@ -7,7 +7,8 @@ 2. Fingerprint — recomputes and compares digest 3. Commit SHA — compares against current repo state 4. Test outcomes — all tests in receipt must have passed; skipped marked - tests are rejected unless allow_skipped is set + tests are rejected unless allow_skipped is set, or an expected-skips + baseline is given and the receipt's skip set matches it EXACTLY 5. GPU info — optionally require environment.gpu_info (require_gpu) 6. Freshness — receipt must not be older than max_age_days 7. Dirty policy — reject dirty-tree receipts if policy requires clean @@ -45,6 +46,17 @@ def _load_policy(policy_path: Optional[str]) -> dict: return yaml.safe_load(text) or {} +def _load_expected_skips(path: str) -> set: + """Baseline file: one node ID per line; blank lines and '#' comments ignored.""" + lines = Path(path).read_text().splitlines() + entries = set() + for line in lines: + line = line.strip() + if line and not line.startswith("#"): + entries.add(line) + return entries + + def _receipt_payload_without_sig(receipt: dict) -> bytes: from .receipt import canonicalize payload = {k: v for k, v in receipt.items() if k != "signature"} @@ -86,6 +98,7 @@ def verify_receipt( allow_unsigned: bool = False, allow_skipped: bool = False, require_gpu: Optional[bool] = None, + expected_skips_path: Optional[str] = None, ) -> bool: try: _verify( @@ -97,6 +110,7 @@ def verify_receipt( allow_unsigned=allow_unsigned, allow_skipped=allow_skipped, require_gpu=require_gpu, + expected_skips_path=expected_skips_path, ) return True except VerificationError as e: @@ -112,6 +126,7 @@ def _verify( max_age_days_override: Optional[int], allow_unsigned: bool = False, allow_skipped: bool = False, + expected_skips_path: Optional[str] = None, require_gpu: Optional[bool] = None, ): from .config import load_toml_defaults @@ -233,12 +248,51 @@ def _verify( f"{len(failed)} test(s) did not pass: {', '.join(failed)}" ) skipped = [t["node_id"] for t in tests if t.get("outcome") == "skipped"] - if skipped and not allow_skipped: + + # Expected-skips baseline: the receipt's skip set must match EXACTLY. + # Stricter than --allow-skipped (which accepts ANY skips): new skips fail, + # and a baselined test that now runs flags the baseline as stale. + expected_skips = None + if expected_skips_path is not None: + if allow_skipped: + raise VerificationError( + "--expected-skips and --allow-skipped are mutually exclusive: " + "the baseline already defines exactly which skips are acceptable." + ) + expected_skips = _load_expected_skips(expected_skips_path) + elif not allow_skipped and toml_cfg.get("expected_skips"): + expected_skips = set(toml_cfg["expected_skips"]) + + if expected_skips is not None: + got = set(skipped) + unexpected = sorted(got - expected_skips) + stale = sorted(expected_skips - got) + problems = [] + if unexpected: + problems.append( + f"{len(unexpected)} skip(s) NOT in the baseline: {', '.join(unexpected)}" + ) + if stale: + problems.append( + f"{len(stale)} baseline entr(y/ies) that did NOT skip (stale " + f"baseline — update it): {', '.join(stale)}" + ) + if problems: + raise VerificationError( + "Skipped tests do not match the expected-skips baseline. " + + " | ".join(problems) + ) + print( + f"[gpu-proof] Skipped tests match the expected baseline " + f"({len(expected_skips)} pinned skip(s))" + ) + elif skipped and not allow_skipped: raise VerificationError( f"{len(skipped)} marked test(s) were skipped: {', '.join(skipped)}. " - "Skipped tests prove nothing; pass --allow-skipped to accept them." + "Skipped tests prove nothing; pass --allow-skipped to accept them, " + "or pin them with --expected-skips BASELINE_FILE." ) - if skipped: + elif skipped: print( f"[gpu-proof] WARNING: {len(skipped)} skipped test(s) accepted " "(--allow-skipped)" diff --git a/tests/conftest.py b/tests/conftest.py index 3e6dc3c..5ced2aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,6 +15,18 @@ ) +@pytest.fixture(autouse=True) +def _no_gh_cli(monkeypatch): + """Keep the suite hermetic: never shell out to `gh` for the signer login. + + On a developer box with an authenticated gh CLI, receipt building would + otherwise hit the network. Signer-resolution tests re-patch explicitly. + """ + monkeypatch.setattr( + "pytest_gpu_proof.receipt.get_gh_cli_login", lambda: None + ) + + @pytest.fixture def ed25519_keypair(): """Fresh Ed25519 keypair for testing — no disk I/O, no real SSH keys needed.""" diff --git a/tests/test_receipt.py b/tests/test_receipt.py index 545094d..8bb502a 100644 --- a/tests/test_receipt.py +++ b/tests/test_receipt.py @@ -96,3 +96,47 @@ def test_write_receipt(tmp_path, mock_config, sample_test_results, signer, tmp_g loaded = json.loads(out.read_text()) assert loaded["schema_version"] == "1" assert "signature" in loaded + + +# ─── signer (github_username) resolution ──────────────────────────────────── + +def _build(mock_config, sample_test_results, tmp_git_repo): + os.chdir(tmp_git_repo) + return build_receipt_payload( + mock_config, sample_test_results, "2026-01-01T00:00:00Z", "2026-01-01T00:00:01Z" + ) + + +def test_signer_uses_gh_cli_login_as_keyholder( + monkeypatch, mock_config, sample_test_results, tmp_git_repo +): + monkeypatch.setattr( + "pytest_gpu_proof.receipt.get_gh_cli_login", lambda: "keyholder" + ) + payload = _build(mock_config, sample_test_results, tmp_git_repo) + assert payload["repo"]["github_username"] == "keyholder" + + +def test_signer_config_beats_gh_cli( + monkeypatch, mock_config, sample_test_results, tmp_git_repo +): + monkeypatch.setattr( + "pytest_gpu_proof.receipt.get_gh_cli_login", lambda: "keyholder" + ) + mock_config.github_username = "configured" + payload = _build(mock_config, sample_test_results, tmp_git_repo) + assert payload["repo"]["github_username"] == "configured" + + +def test_signer_remote_owner_fallback_warns( + monkeypatch, capsys, mock_config, sample_test_results, tmp_git_repo +): + # gh CLI unavailable (autouse fixture) and the origin remote is org-owned + import subprocess + subprocess.run( + ["git", "remote", "add", "origin", "git@github.com:Some-Org/repo.git"], + cwd=tmp_git_repo, check=True, capture_output=True, + ) + payload = _build(mock_config, sample_test_results, tmp_git_repo) + assert payload["repo"]["github_username"] == "Some-Org" + assert "origin remote owner" in capsys.readouterr().out diff --git a/tests/test_verifier.py b/tests/test_verifier.py index 385834f..e10106c 100644 --- a/tests/test_verifier.py +++ b/tests/test_verifier.py @@ -281,3 +281,83 @@ def test_verify_fails_on_failed_test(tmp_path, tmp_git_repo, signer_with_key): with _mock_github_keys(public_key): with pytest.raises(VerificationError, match="did not pass"): _verify(str(path), None, str(tmp_git_repo), "testuser", None) + + +# ─── expected-skips baseline ──────────────────────────────────────────────── + +def _mixed_results(): + return [ + {"node_id": "tests/test_add.py::test_add", "outcome": "passed", + "duration_s": 0.01, "checks": []}, + {"node_id": "tests/test_add.py::test_skipped", "outcome": "skipped", + "duration_s": 0.0, "checks": []}, + ] + + +def test_expected_skips_exact_match_passes(tmp_path, tmp_git_repo, signer_with_key): + signer, public_key = signer_with_key + path = _make_receipt(tmp_path, tmp_git_repo, signer, results=_mixed_results()) + baseline = tmp_path / "expected_skips.txt" + baseline.write_text( + "# known-legitimate skips\n" + "\n" + "tests/test_add.py::test_skipped\n" + ) + with _mock_github_keys(public_key): + _verify( + str(path), None, str(tmp_git_repo), "testuser", None, + expected_skips_path=str(baseline), + ) + + +def test_expected_skips_rejects_unexpected_skip(tmp_path, tmp_git_repo, signer_with_key): + signer, public_key = signer_with_key + path = _make_receipt(tmp_path, tmp_git_repo, signer, results=_mixed_results()) + baseline = tmp_path / "expected_skips.txt" + baseline.write_text("tests/test_add.py::test_other_skip\n") + with _mock_github_keys(public_key): + with pytest.raises(VerificationError, match="NOT in the baseline"): + _verify( + str(path), None, str(tmp_git_repo), "testuser", None, + expected_skips_path=str(baseline), + ) + + +def test_expected_skips_rejects_stale_baseline(tmp_path, tmp_git_repo, signer_with_key): + signer, public_key = signer_with_key + # all tests pass — the baseline entry no longer skips + path = _make_receipt(tmp_path, tmp_git_repo, signer) + baseline = tmp_path / "expected_skips.txt" + baseline.write_text("tests/test_add.py::test_skipped\n") + with _mock_github_keys(public_key): + with pytest.raises(VerificationError, match="stale"): + _verify( + str(path), None, str(tmp_git_repo), "testuser", None, + expected_skips_path=str(baseline), + ) + + +def test_expected_skips_mutually_exclusive_with_allow_skipped( + tmp_path, tmp_git_repo, signer_with_key +): + signer, public_key = signer_with_key + path = _make_receipt(tmp_path, tmp_git_repo, signer, results=_mixed_results()) + baseline = tmp_path / "expected_skips.txt" + baseline.write_text("tests/test_add.py::test_skipped\n") + with _mock_github_keys(public_key): + with pytest.raises(VerificationError, match="mutually exclusive"): + _verify( + str(path), None, str(tmp_git_repo), "testuser", None, + allow_skipped=True, expected_skips_path=str(baseline), + ) + + +def test_expected_skips_from_toml(tmp_path, tmp_git_repo, signer_with_key): + signer, public_key = signer_with_key + path = _make_receipt(tmp_path, tmp_git_repo, signer, results=_mixed_results()) + (tmp_git_repo / "pyproject.toml").write_text( + "[tool.gpu_proof]\n" + 'expected_skips = ["tests/test_add.py::test_skipped"]\n' + ) + with _mock_github_keys(public_key): + _verify(str(path), None, str(tmp_git_repo), "testuser", None)