diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c1a366..bd63e73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,6 +140,50 @@ jobs: exit 1 fi + # plan/00-SPINE.md S12 permits Python in exactly three places, and the + # evaluation harness is one of them. eval/ is a pure-Python tree. + # + # The register this job validates is the artifact plan step M0.18 reads to + # decide whether Anvil's detection-model tier exists at all, so the property + # that matters is narrow and absolute: a row whose experiment has not run must + # never be readable as a pass. The schema encodes that; this job is what makes + # anything actually run the schema. It shipped with nothing in the repository + # checking one against the other, and a schema nobody runs is a comment. + eval-harness: + name: Eval harness (Python) + runs-on: ubuntu-latest + defaults: + run: + working-directory: eval + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + # [dev] carries pytest/ruff. Core deps are deliberately minimal -- the + # heavy [stats] and [models] extras are pulled by the packets that need + # them (M0.6, M0.10, M0.15), not here. + - name: Install + run: python -m pip install --upgrade pip && python -m pip install -e ".[dev]" + + - name: Lint + run: python -m ruff check . + + - name: Tests, including the register mutation suite + run: python -m pytest tests/ -q + + # A skipped test is a legitimate outcome here -- M0.7's smoke path is + # UNVERIFIED by design because nothing downloads the opengrep engine in + # CI. But a suite that silently degrades to all-skips would still report + # green, so assert the register suite actually ran. + - name: The register mutation suite really ran + run: | + out=$(python -m pytest tests/test_register_schema.py -q 2>&1) + echo "$out" + echo "$out" | grep -qE '[0-9]+ passed' || { + echo "::error::register schema suite reported no passing tests"; exit 1; } + # NOTE: there is deliberately no "plan" job here. The planning workspace # (plan/) and its verification tooling (tools/) are local-only and gitignored, # same posture as research/ -- they are how Anvil was designed and verified, diff --git a/eval/.gitignore b/eval/.gitignore new file mode 100644 index 0000000..b46d6f3 --- /dev/null +++ b/eval/.gitignore @@ -0,0 +1,81 @@ +# eval/.gitignore — Anvil evaluation harness (M0.2) +# +# The repository-root .gitignore already covers the generic Python noise +# (__pycache__/, *.egg-info/, .venv/) and blanket-ignores `models/`, +# `*.gguf` and `*.safetensors`. This file adds the eval-tree-specific rules +# for model and dataset caches, and re-includes the provenance metadata that +# the Milestone 0 register depends on. +# +# Deeper .gitignore files take precedence over shallower ones, so the +# re-inclusions below override the root file's `models/` rule for eval/models. + +# --- Dataset payloads (M0.3 PrimeVul, M0.4 ARVO, M0.5 CWE-Bench-Java) ------- +# Corpora are acquired locally and never committed or re-hosted: +# PrimeVul is Google-Drive-gated (M0.3 forbidden actions) and ARVO / +# CWE-Bench-Java are pulled from their upstream repos at a pinned SHA. +data/** +# ...but keep the directory skeleton, the MANIFEST files that record the +# acquisition-date snapshot and pinned SHAs, and any prose notes. +!data/**/ +!data/**/MANIFEST +!data/**/MANIFEST.* +!data/**/*.md +!data/**/.gitkeep + +# --- Model artifacts (M0.6) ------------------------------------------------- +# Weights are never committed. Only the pinned-download manifest, the exact HF +# revision SHA, the archived LICENSE text (S8 compliance mechanics) and the +# checksums are tracked. +!models/ +models/** +!models/**/ +!models/**/MANIFEST +!models/**/MANIFEST.* +!models/**/*.md +!models/**/*.json +!models/**/*.txt +!models/**/*.sha256 +!models/**/LICENSE* +!models/**/.gitkeep +# Belt and braces: never track a weight file even if a rule above widens. +models/**/*.gguf +models/**/*.safetensors +models/**/*.onnx +models/**/*.onnx_data +models/**/*.bin +models/**/*.pt +models/**/*.pth + +# --- Tool binaries (M0.7 opengrep engine + pinned ruleset) ------------------ +# The repository-root .gitignore blanket-ignores `tools/` (unanchored, so it +# matches eval/tools/ too). That rule is meant for the root-level planning +# tooling; eval/tools/ is M0.7's write scope and its pin records must be +# tracked, so re-include the directory here and exclude only the payloads. +!tools/ +tools/**/bin/ +tools/**/*.exe +tools/**/*.tar.gz +tools/**/*.zip +tools/**/*.tgz +tools/**/*.whl + +# --- Hugging Face / dataset download caches -------------------------------- +.hf_cache/ +.cache/ +hf_home/ +transformers_cache/ +datasets_cache/ + +# --- Local run scratch ------------------------------------------------------ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.coverage +.coverage.* +htmlcov/ +scratch/ +*.log + +# NOTE: eval/results/ is deliberately NOT ignored — the Milestone 0 exit +# criteria require every register row's `artifact_path` to point at a real, +# committed eval/results/.json file. diff --git a/eval/pyproject.toml b/eval/pyproject.toml new file mode 100644 index 0000000..ac5698e --- /dev/null +++ b/eval/pyproject.toml @@ -0,0 +1,81 @@ +[build-system] +requires = ["setuptools>=77.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "anvil-eval" +version = "0.1.0" +description = "Anvil Milestone 0 evaluation harness — experiment register, corpora loaders, and experiment runners." +requires-python = ">=3.11" +license = "Apache-2.0" +authors = [{ name = "Anvil maintainers" }] +keywords = ["vulnerability", "evaluation", "harness", "anvil", "milestone-0"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Security", + "Private :: Do Not Upload", +] + +# Core runtime dependencies. Kept deliberately small so that +# `pip install -e eval/` resolves quickly in a fresh virtualenv with no +# network access beyond the package index (M0.2 stop condition). +# Anything heavy, platform-specific, or only needed by one later M0 step +# lives in an optional-dependency group below. +dependencies = [ + "jsonschema>=4.23", # M0.1/M0.17 register validation against eval/schema/register.schema.json + "pyyaml>=6.0.2", # eval/register.yaml read/write + "numpy>=2.1", # bootstrap confidence intervals (EXP-01, EXP-02) + "requests>=2.32", # pinned-artifact fetches, S12-RTT HTTP client +] + +[project.optional-dependencies] +# EXP-02 code-metrics logistic-regression baseline. +stats = [ + "scikit-learn>=1.5", + "scipy>=1.14", +] +# M0.6 model acquisition and the S12-RTT ONNX encoder worker. +# Not installed by default: these wheels are large and platform-specific, +# and M0.2 is forbidden from vendoring any model weight. +models = [ + "huggingface-hub>=0.26", + "onnxruntime>=1.20", + "tokenizers>=0.20", + "transformers>=4.46", +] +dev = [ + "pytest>=8.3", + "pytest-cov>=5.0", + "ruff>=0.7", +] + +[tool.setuptools] +package-dir = { "" = "src" } + +[tool.setuptools.packages.find] +where = ["src"] +include = ["anvil_eval*"] + +[tool.pytest.ini_options] +minversion = "8.0" +testpaths = ["tests"] +addopts = "-ra --strict-markers --strict-config" +markers = [ + "network: test reaches the public internet; skipped in offline runs", + "docker: test shells out to Docker (ARVO reproducers, EXP-04)", + "model: test loads model weights from eval/models/", + "slow: test takes more than a few seconds", +] + +[tool.ruff] +line-length = 100 +src = ["src", "tests"] +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "B", "UP"] diff --git a/eval/register.yaml b/eval/register.yaml new file mode 100644 index 0000000..dfeb35c --- /dev/null +++ b/eval/register.yaml @@ -0,0 +1,758 @@ +# Anvil — Milestone 0 experiment register +# +# Authored by step M0.1 (plan/10-milestone0-evaluation.md lines 33-58). +# Field list: plan/10-milestone0-evaluation.md "## Experiment Register Schema". +# Row content: plan/10-milestone0-evaluation.md "## Go/No-Go Decision Table". +# Validates against: eval/schema/register.schema.json +# +# --------------------------------------------------------------------------- +# HOW TO READ THE `decision` FIELD — read this before acting on any row. +# +# `decision: UNRESOLVED` means NOBODY HAS DECIDED YET. It is NOT a pass. +# It is NOT a "probably fine". It is the absence of +# a decision, stated out loud. +# +# Every row in this file is born UNRESOLVED and stays UNRESOLVED until an +# orchestrator-inline step writes PASS / FAIL / AMBIGUOUS / DEFERRED into it. +# There is no null, no empty string and no missing-key representation of an +# undecided row — the schema forbids all three — precisely so that a row whose +# decision was never recorded cannot be skimmed as if it had passed. +# +# Worker packets populate `result` only. They NEVER touch `decision`. +# (plan/10-milestone0-evaluation.md, note under "## Experiment Register Schema": +# "`decision` is only ever set by an orchestrator-inline step ... never by a +# worker packet".) +# +# EXP-01 and EXP-02 are the two kill criteria (plan/00-SPINE.md S3.1, S3.2). +# Step M0.18 reads their rows and decides whether Anvil's small-model detection +# tier exists at all. If either row still reads UNRESOLVED, M0.18 has no input +# and no Milestone-1 component-build packet may be dispatched. +# --------------------------------------------------------------------------- +# +# STATE AT AUTHORING (M0.1): nothing has been run. +# - 14/14 rows decision: UNRESOLVED +# - 6 rows status: not_started (EXP-01..EXP-04, INSTR-01, S12-RTT) +# - 8 rows status: deferred (EXP-05..EXP-12) +# - 14/14 rows result: all fields null +# - `owning_future_phase` is null on every deferred row. Step M0.17 owns +# populating it; the milestone exit criteria require it non-null before M0 +# closes. Null here means "not yet assigned", not "no owner needed". + +version: 1 + +experiments: + + # ========================================================================= + # KILL CRITERION #1 + # ========================================================================= + - id: EXP-01 + name: Advisory-permutation ablation (Test 4) + kind: experiment + spine_ref: S3.1 + source_ref: >- + research/03-detection-training-data-and-method.md — "Evaluation harness" table, Test 4 row; + enumerated as experiment #1 in research/14-critique-and-gaps.md B7 table + owner_step: M0.9 + status: not_started + decision_gated: >- + Whether the detection model actually reads the advisory, i.e. whether advisory-conditioning is + real. PASS: proceed with the three-tier design (research/02) using this candidate model. + FAIL: kill criterion #1 — CVE ingestion is decoration (plan/00-SPINE.md S3.1); delete + advisory-text conditioning from the SAST adjudicator's design, restrict advisory text to the + narrow vendored/backported-code scope only (S1 corrected requirement #8; research/14 B2 fix), + and re-evaluate whether Tier 3 should exist at all pending EXP-02. + corpus: + - name: PrimeVul-Paired + licence: MIT + licence_ref: >- + research/03-detection-training-data-and-method.md S2, via + plan/10-milestone0-evaluation.md "## Pinned Versions And Licences" + - name: Qwen/Qwen3.5-2B (primary candidate) + licence: Apache-2.0 — pending text-only-variant confirmation (M0.6/M0.8) + licence_ref: plan/10-milestone0-evaluation.md "## Pinned Versions And Licences" + - name: Gemma-4 (specific variant TBD, secondary candidate) + licence: >- + Apache-2.0 for the specific variant only — re-verify per-variant, do not infer from the + family-level reversal + licence_ref: >- + plan/00-SPINE.md S4 caveat, via plan/10-milestone0-evaluation.md + "## Pinned Versions And Licences" + method: >- + Substitute an unrelated advisory for the correct one and measure the EXHIBITS -> + DOES_NOT_EXHIBIT verdict flip rate over PrimeVul-Paired. Run against Qwen3.5-2B (primary) and + Gemma-4 (secondary) and report per-model breakdown, not a single aggregate — the two candidates + may diverge and that divergence is itself model-selection evidence. The permutation must be + drawn from the full advisory pool, never a CWE-matched subset (a CWE-matched subset understates + the true flip rate and biases toward a false PASS; M0.12 critic gate). + metric: >- + Flip rate (EXHIBITS -> DOES_NOT_EXHIBIT) on the PrimeVul-Paired test split, reported with a + bootstrap 95% confidence interval — n_pairs is only 564, so CI width matters at this sample size. + threshold_pass: >- + Flip rate > 80% (plan/00-SPINE.md S3.1; research/03-detection-training-data-and-method.md + Test 4 row). + threshold_fail: >- + Flip rate < 50% (plan/00-SPINE.md S3.1) — CVE ingestion is decoration. The 50-80% band is + AMBIGUOUS; PROPOSED handling is to run the QLoRA runner-up (research/03) before a final call, + and plan/10-milestone0-evaluation.md "## Open Questions" requires M0.18's orchestrator to set an + explicit escalation policy for this band before it can occur in practice. + reference_baseline: >- + research/03-detection-training-data-and-method.md — "Evaluation harness" table, Test 4 row + (bar: > 80% flip rate) + result: + value: null + unit: null + measured_at: null + artifact_path: null + decision: UNRESOLVED + depends_on: [] + deferred_reason: null + owning_future_phase: null + + # ========================================================================= + # KILL CRITERION #2 + # ========================================================================= + - id: EXP-02 + name: Code-metrics logistic-regression baseline + kind: experiment + spine_ref: S3.2 + source_ref: >- + research/02-small-detection-models.md finding B / S3 (arXiv:2509.19117); + enumerated as experiment #2 in research/14-critique-and-gaps.md B7 table + owner_step: M0.10 + status: not_started + decision_gated: >- + Whether the small-model detection tier should exist at all. PASS (logistic regression is + significantly worse than the model on both metrics — non-overlapping CIs, LR lower): the model + tier is justified. FAIL: kill criterion #2 — the model tier does not exist (plan/00-SPINE.md + S3.2); delete Tier 3 (small-model adjudicator) entirely and ship Tier 1 (deterministic + pre-filter) plus optional Tier 2 (encoder ranker, or the LR itself) only — research/02's + three-tier design demoted to two. + corpus: + - name: PrimeVul / PrimeVul-Paired + licence: MIT + licence_ref: >- + research/03-detection-training-data-and-method.md S2, via + plan/10-milestone0-evaluation.md "## Pinned Versions And Licences" + - name: Qwen/Qwen3.5-2B (model arm, shared with EXP-01) + licence: Apache-2.0 — pending text-only-variant confirmation (M0.6/M0.8) + licence_ref: plan/10-milestone0-evaluation.md "## Pinned Versions And Licences" + method: >- + Train a logistic regression over classic code metrics (cyclomatic complexity, LoC, nesting + depth, etc.; full feature list recorded for reproducibility) on PrimeVul and score it against + the small model's arm on the identical held-out split and identical metric definitions used by + EXP-01, so the comparison is apples-to-apples. Two hard constraints: the LR must not be trained + on the paired test split used for evaluation (no leakage), and the training set must not be + class-balanced — research/03 and research/14 (S14 citation) both name balanced training as the + specific failure mode that manufactures false confidence (the 0.09-precision catastrophe). + Both arms reported with bootstrap CIs. + metric: >- + Test-1 pairwise-correct (P-C) rate and Test-2 precision at recall 0.70, computed for both the + LR arm and the model arm, each with a bootstrap 95% confidence interval. + threshold_pass: >- + The LR is significantly worse than the small model on BOTH metrics — non-overlapping 95% CIs + with the LR lower. PROPOSED — no published Anvil-specific equivalence margin exists + (research/02-small-detection-models.md S3, arXiv:2509.19117; plan/00-SPINE.md S3.2). + threshold_fail: >- + The LR's Test-1 P-C and Test-2 precision-at-recall-0.70 fall within or exceed the small model's + 95% CI. Note plan/10-milestone0-evaluation.md "## Open Questions": this equivalence margin is an + engineering judgment call, not a citation — the statistical protocol (sample size, CI method) + must be confirmed before EXP-02 runs. + reference_baseline: >- + research/03-detection-training-data-and-method.md — "Evaluation harness" table: Test 1 bar + P-C > 25% with reference GPT-4 CoT 12.94%; Test 2 bar F1 > 5.22% (flag-everything) with + reference best fine-tune 5.82% + result: + value: null + unit: null + measured_at: null + artifact_path: null + decision: UNRESOLVED + depends_on: + - EXP-01 + deferred_reason: null + owning_future_phase: null + + # ========================================================================= + - id: EXP-03 + name: llama-bench prefill on real hardware + kind: experiment + spine_ref: S2, S9 + source_ref: >- + research/02-small-detection-models.md S8 and research/05-inference-serving-and-hardware.md; + enumerated as experiment #3 in research/14-critique-and-gaps.md B7 table + owner_step: M0.11 + status: not_started + decision_gated: >- + Every downstream compute claim in the plan. PASS: S2's ~17 min / ~2 min budget and S9's tier + assignments stand as written. FAIL: re-derive the candidates-per-scan ceiling from INSTR-01's + measured count against the actual throughput; if the recomputed wall-clock exceeds S2's budget, + drop to Qwen3.5-0.8B, move Tier 3 to the encoder-only ranker (research/02 flip condition #2), + or raise the minimum hardware tier for the detection path. + corpus: + - name: Qwen/Qwen3.5-2B (primary candidate) + licence: Apache-2.0 — pending text-only-variant confirmation (M0.6/M0.8) + licence_ref: plan/10-milestone0-evaluation.md "## Pinned Versions And Licences" + - name: Gemma-4 (specific variant TBD, secondary candidate) + licence: Apache-2.0 for the specific variant only — re-verify per-variant + licence_ref: >- + plan/00-SPINE.md S4 caveat, via plan/10-milestone0-evaluation.md + "## Pinned Versions And Licences" + - name: HuggingFaceTB/SmolLM3-3B (auditability runner-up) + licence: Apache-2.0 + licence_ref: research/02-small-detection-models.md S13 + - name: llama-bench (ggml-org/llama.cpp, pinned release tag) + licence: MIT + licence_ref: plan/10-milestone0-evaluation.md "## Pinned Versions And Licences" + method: >- + llama-bench prefill sweep for each candidate model at realistic pair-prompt length (~2,000 + tokens: ~800 advisory/CWE text + ~1,000 code chunk + ~200 instruction), on whatever S9 hardware + tier is actually available, recorded against the tier it represents. Raw llama-bench output log + retained and the quantisation level used (Q4_K_M or otherwise) recorded — the corpus's own + "<1% quality loss for Q4" claim is itself flagged unverified for Anvil's exact task + (research/02 Risks). No extrapolation from another model family's numbers: the entire point is + that no Qwen3.5/Gemma-4 CPU throughput number has been measured anywhere in the corpus. + metric: >- + Prefill throughput in tokens/second at a 2,000-token prompt — this is the headline number, + because Anvil is prefill-bound (research/02 S8, research/14 B1). Generation throughput is + recorded but is explicitly NOT the headline. + threshold_pass: >- + Measured prefill >= 600 tok/s on Tier-S hardware for the ~2B candidate. PROPOSED, informed by + published scaling: the non-generous parameter-scaled estimate in research/14-critique-and-gaps.md + B1, scaling research/02 S8's measured 170 tok/s at 7B. (The "2,000 tok/s" figure used elsewhere + in the corpus is explicitly "a 3.3x gift to the design" and is not the bar.) + threshold_fail: >- + Measured prefill < 600 tok/s on Tier-S for the ~2B candidate — triggers the re-derivation of + the candidates-per-scan ceiling described in decision_gated. + reference_baseline: >- + research/02-small-detection-models.md S8 (Malakhov, CEUR-WS Vol-4164) — the measured statement + is "a 256-token prompt was about 6.5 s on E5-2695 v2 for a 7B model, 1.5 s on the new Xeon, and + 3 s on the laptop". The "~170 tok/s prefill at 7B" figure is the arithmetic derivation + (256 tok / 1.5 s), not a number printed in the source; the "Xeon Platinum 8480+" SKU is named by + plan/10-milestone0-evaluation.md, not by research/02, which says only "the new Xeon". + result: + value: null + unit: null + measured_at: null + artifact_path: null + decision: UNRESOLVED + depends_on: [] + deferred_reason: null + owning_future_phase: null + + # ========================================================================= + - id: EXP-04 + name: Open-weight vs. frontier patch quality (ARVO / CWE-Bench-Java) + kind: experiment + spine_ref: S3, S7 + source_ref: >- + research/10-prior-art-and-landscape.md, research/04-remediation-coding-model.md and + research/11-fix-validation-and-false-positives.md S7; enumerated as experiment #4 in + research/14-critique-and-gaps.md B7 table + owner_step: M0.16 + status: not_started + decision_gated: >- + Anvil's claimed central differentiator — the "possibly better system" framing in + plan/00-SPINE.md S3. PASS: the framing is supported; no redesign forced, since S7's + never-auto-merge posture already assumes a low verified-fix rate. FAIL: the coding-agent tier + needs a larger or different open-weight model, or the differentiator claim in project messaging + should be revised. Either way this does not change the safety posture — S7 already mandates + human review regardless of rate. + corpus: + - name: ARVO (n132/ARVO repo commit SHA + specific n132/arvo:-vul Docker tags) + licence: BSD-2-Clause + licence_ref: >- + research/16-fuzzing-and-dynamic-analysis.md S2/S25 — cite only the canonical 2024-08 paper + (arXiv:2408.02153); the 2026 posting is withdrawn + - name: CWE-Bench-Java (iris-sast/cwe-bench-java commit SHA) + licence: MIT + licence_ref: plan/spine-b-open-licences.md section 7 + - name: Qwen/Qwen3-Coder-Next (coding-agent arm) + licence: Apache-2.0 + licence_ref: >- + research/14-critique-and-gaps.md "Held up exactly" table (79.7B params, 512 experts, + 10 active) + method: >- + Run the open-weight coding agent over the sampled ARVO and CWE-Bench-Java subsets using the + AutoPatchBench-style three-stage validation gate (crash input no longer crashes -> 10-minute + fuzz survival -> differential behaviour check) and ARVO's reproduce() contract. The frontier + comparison arm is taken from published literature, never re-run: plan/00-SPINE.md S0 requires + self-hosted open-weight models only, so no proprietary frontier API is called. The exploit + oracle (research/11 gate 9) is mandatory — a patch is never counted "fixed" on test-suite pass + alone, because research/11's Vul4J finding is that 10.3% of patches pass tests while remaining + exploitable, and ARVO itself found 300+ falsely-patched still-active vulnerabilities in + OSS-Fuzz's own "fixed" set. Upstream "fixed" labels are never trusted as ground truth. + metric: >- + Verified-fix rate per corpus (fraction of cases clearing all three gates with a recorded + exploit-oracle re-execution), plus generation success rate and the full gate-failure breakdown + (compile_fail, test_fail, exploit_still_triggers, deceptive_pass) — never an aggregate + percentage alone. + threshold_pass: >- + Verified-fix rate >= 5% on ARVO, matching or exceeding Gemini 1.5 Pro's AutoPatchBench-verified + 5.3%. PROPOSED — derived by analogy, not a number published for Qwen3-Coder-Next specifically; + plan/10-milestone0-evaluation.md "## Open Questions" requires explicit sign-off before this + drives any messaging claim. + threshold_fail: >- + Verified-fix rate < 2% on ARVO. PROPOSED, same caveat as threshold_pass. + reference_baseline: >- + research/11-fix-validation-and-false-positives.md S7 — AutoPatchBench: Gemini 1.5 Pro 61.1% + generation / 5.3% fully-verified; supporting context research/16-fuzzing-and-dynamic-analysis.md + S18 (DARPA AIxCC 68% of synthetic vulnerabilities patched, ~$152/task) and plan/00-SPINE.md S7 + ("Best measured security-patch rate on real CVEs is 34.0%") + result: + value: null + unit: null + measured_at: null + artifact_path: null + decision: UNRESOLVED + depends_on: [] + deferred_reason: null + owning_future_phase: null + + # ========================================================================= + # PERMANENT INSTRUMENT — not a kill gate + # ========================================================================= + - id: INSTR-01 + name: Candidates-per-scan (permanent instrument) + kind: instrumentation + spine_ref: S2 + source_ref: >- + plan/00-SPINE.md S2 ("Instrument it on day one"); research/14-critique-and-gaps.md B1 + and its "Suggested fix" + owner_step: M0.14 + status: not_started + decision_gated: >- + The single number plan/00-SPINE.md S2 says decides whether Anvil is affordable at all — "it, + not model size, determines feasibility". PASS: affordability confirmed at the current + recall-tier tuning; carry the measured count into Milestone 1's scheduler design. FAIL: + re-scope the recall tier — narrower rules, CWE-class routing, or tighter version-range matching + (research/14-critique-and-gaps.md B1 "Suggested fix") — before Milestone 1 proceeds. This row + is a permanent instrument, not a kill gate; it is re-run, not retired. + corpus: + - name: AikidoSec/opengrep-rules (pinned commit SHA, diffed before any future promotion) + licence: MIT + licence_ref: research/14-critique-and-gaps.md M6 + - name: opengrep engine (pinned release version, subprocess invocation only) + licence: LGPL-2.1 + licence_ref: plan/00-SPINE.md S4 + method: >- + A re-runnable instrument (a standalone callable, never a throwaway script or notebook cell — + S2 asks for a permanent day-one measurement) that runs pinned opengrep plus + AikidoSec/opengrep-rules against a target repo and counts candidates emitted per full scan and + per push. Exercised in M0 against at least 3 structurally different sample repos (e.g. a small + CLI tool, a web-app-shaped repo, a library) so the count is not an artifact of one repo's shape. + metric: >- + candidates_full_scan and candidates_per_push per target repo, recorded together with the + ruleset commit SHA. + threshold_pass: >- + Fewer than 500 candidates per full scan AND fewer than 50 per push (plan/00-SPINE.md S2, exact + figures) — at which the model tier costs ~17 min and ~2 min respectively. + threshold_fail: >- + Counts "in the thousands" (plan/00-SPINE.md S2) — the design does not work as written and must + be re-scoped. + reference_baseline: >- + research/14-critique-and-gaps.md B1, restating plan/00-SPINE.md S2's figures: < 500 per full + scan and < 50 per push + result: + value: null + unit: null + measured_at: null + artifact_path: null + decision: UNRESOLVED + depends_on: [] + deferred_reason: null + owning_future_phase: null + + # ========================================================================= + - id: S12-RTT + name: Encoder HTTP round-trip under realistic pair volume + kind: experiment + spine_ref: S12 + source_ref: >- + plan/00-SPINE.md S12 — the recorded counter-argument to the Go control-plane pick: "Measure + encoder RTT under realistic pair volume during Milestone 0 before this is irreversible" + owner_step: M0.15 + status: not_started + decision_gated: >- + Whether S12's Go-for-the-control-plane pick survives contact with a measured number. PASS: the + pick is confirmed; no action needed. FAIL: record as a "Conflict With Spine" item for the + orchestrator — collapsing the encoder and the orchestrator into one Python process becomes the + stronger pick per S12's own stated counter-argument. Note that M0 only produces the number; per + plan/10-milestone0-evaluation.md "## Conflicts With Spine" the resulting decision belongs to + whichever future step owns S12, not to M0. + corpus: + - name: microsoft/unixcoder-base (Tier-2 encoder candidate) + licence: Apache-2.0 weights / MIT repo + licence_ref: research/02-small-detection-models.md S18 + - name: ONNX Runtime (pinned release version, encoder worker) + licence: MIT + licence_ref: plan/10-milestone0-evaluation.md "## Pinned Versions And Licences" + method: >- + Serve the Tier-2 encoder candidate via ONNX Runtime as a separate always-on process addressed + over HTTP — never an in-process library call, since the in-process case is exactly what S12 + forbids and measuring it would answer the wrong question — and measure aggregate round-trip + time across a realistic pair volume ("thousands of advisory x code-chunk pairs per scan", S12's + own framing), at a pair count at least as large as INSTR-01's measured candidates-per-scan for + the sample repos used. The pair count and its relationship to INSTR-01's number must be stated + explicitly so the overhead-fraction calculation is traceable. + metric: >- + Total RTT (seconds) and RTT per pair (ms) at the measured pair count, expressed as a fraction + of the per-scan model-tier wall clock derived from EXP-03's prefill throughput x INSTR-01's + candidate count. + threshold_pass: >- + Aggregate encoder overhead <= 20% (PROPOSED) of the per-scan model-tier wall clock established + by EXP-03 x INSTR-01. + threshold_fail: >- + Aggregate encoder overhead > 20% (PROPOSED) of that wall clock. The spine explicitly calls this + "a cost never measured" (plan/00-SPINE.md S12), so the 20% figure is an engineering proposal, + not a sourced bar. + reference_baseline: PROPOSED — no published baseline + result: + value: null + unit: null + measured_at: null + artifact_path: null + decision: UNRESOLVED + depends_on: + - EXP-03 + - INSTR-01 + deferred_reason: null + owning_future_phase: null + + # ========================================================================= + # DEFERRED — the eight named experiments M0 schedules but does not run. + # Each keeps a threshold. None is "not applicable". + # ========================================================================= + - id: EXP-05 + name: Batch-size vs. validated-fix-rate curve + kind: experiment + spine_ref: S7 + source_ref: >- + research/24-coding-agent-consumption.md; enumerated as experiment #5 in + research/14-critique-and-gaps.md B7 table ("the consumption protocol's core parameter") + owner_step: deferred + status: deferred + decision_gated: >- + The remediation consumption protocol's core parameter — how many fix PRs may be in flight + before acceptance rate collapses. PASS: batch size tuned to keep PR acceptance near the ~15% + pre-slop order of magnitude. FAIL: reduce batch size / rate limit further. + corpus: [] + method: >- + Sweep batch size against validated-fix rate and observed PR acceptance rate, against gate 16's + default ceiling of <= 3 open PRs per repo. + metric: Validated-fix rate and PR acceptance rate as a function of batch size. + threshold_pass: >- + PR acceptance rate stays near the ~15% pre-slop order of magnitude + (research/11-fix-validation-and-false-positives.md S22). PROPOSED — no published baseline; this + is a qualitative floor, not a measured bar. + threshold_fail: >- + Acceptance rate falls toward the < 5% post-slop regime research/11 S22 records for curl's + bounty programme once the noise ratio inverted. + reference_baseline: >- + research/11-fix-validation-and-false-positives.md S22 — curl bounty confirmation rate fell + below 5% once the noise ratio inverted; ~15% was the pre-slop baseline + result: + value: null + unit: null + measured_at: null + artifact_path: null + decision: UNRESOLVED + depends_on: [] + deferred_reason: >- + Milestone 0 executes only EXP-01..EXP-04, INSTR-01 and S12-RTT + (plan/10-milestone0-evaluation.md "## Overview"). This experiment measures a parameter of the + remediation consumption protocol, which does not exist yet — there is no PR pipeline to batch. + owning_future_phase: null + + # ========================================================================= + - id: EXP-06 + name: SAST<->DAST correlation precision/recall + kind: experiment + spine_ref: S6 + source_ref: >- + research/18-unified-audit-record.md; enumerated as experiment #6 in + research/14-critique-and-gaps.md B7 table ("whether correlation can be trusted at all") + owner_step: deferred + status: deferred + decision_gated: >- + Whether SAST<->DAST correlation is trustworthy. PASS: keep the correlationGuid-based design + (plan/00-SPINE.md S6). FAIL: treat SAST and DAST as independently-sealed halves with no + automatic linkage claim — already the S6 / research/14 B3 fallback, so a FAIL degrades the + design rather than breaking it. + corpus: [] + method: >- + Measure correlation precision and recall against a seeded corpus in which the true + SAST-finding-to-DAST-finding mapping is known by construction. + metric: Precision and recall of the correlation linkage against the seeded ground truth. + threshold_pass: >- + PROPOSED — no published baseline; the success criterion is to be defined against a seeded + corpus once plan/00-SPINE.md S6's record schema exists. + threshold_fail: >- + PROPOSED — undefined until the S6 schema exists. Not permitted to run without a numeric bar + agreed in advance. + reference_baseline: PROPOSED — no published baseline + result: + value: null + unit: null + measured_at: null + artifact_path: null + decision: UNRESOLVED + depends_on: [] + deferred_reason: >- + Its success criterion is explicitly "to be defined against a seeded corpus once S6's schema + exists" (plan/10-milestone0-evaluation.md "## Go/No-Go Decision Table", EXP-06 row). The S6 + record schema does not exist at Milestone 0, so no threshold can be set and the experiment + cannot be run without inventing one. + owning_future_phase: null + + # ========================================================================= + - id: EXP-07 + name: Static route extraction vs. runtime spec probe + kind: experiment + spine_ref: S4 + source_ref: >- + research/22-attack-surface-discovery.md; enumerated as experiment #7 in + research/14-critique-and-gaps.md B7 table ("whether the DAST inventory design works"); + reporting form from research/14-critique-and-gaps.md m6 + owner_step: deferred + status: deferred + decision_gated: >- + Whether plan/00-SPINE.md S4's declared discovery ordering (runtime probe -> repo specs -> + static extraction -> browser crawl last) is evidence-backed. PASS: the ordering stands as + written. FAIL: reorder or drop the weakest-evidenced stage. + corpus: [] + method: >- + Compare routes recovered by static extraction against those recovered by a runtime spec probe + on the same targets, reporting the numerator and the provenance mix rather than a coverage + percentage — research/14 m6 rejects a coverage percentage as the reporting form, because the + denominator (true route count) is unknowable. + metric: >- + Routes discovered per stage plus provenance mix (which stage first found each route). Not a + coverage percentage. + threshold_pass: >- + PROPOSED — no published baseline. Report the numerator and provenance mix, not a coverage + percentage (research/14-critique-and-gaps.md m6). + threshold_fail: >- + PROPOSED — a stage whose provenance contribution is negligible against its cost is the + "weakest-evidenced stage" to be reordered or dropped. + reference_baseline: PROPOSED — no published baseline + result: + value: null + unit: null + measured_at: null + artifact_path: null + decision: UNRESOLVED + depends_on: [] + deferred_reason: >- + Milestone 0 executes only the six items named in plan/10-milestone0-evaluation.md "## Overview". + This experiment measures the DAST attack-surface inventory, which per plan/00-SPINE.md + S9-AMENDED ships as a separate distribution artifact and has no implementation at M0 to measure. + owning_future_phase: null + + # ========================================================================= + - id: EXP-08 + name: Grammar-constrained DAST confirmation eval + kind: experiment + spine_ref: S4 + source_ref: >- + research/17-dast-model-selection.md; enumerated as experiment #8 in + research/14-critique-and-gaps.md B7 table ("whether the DAST model can be 4B not 9B"); + informed by research/14-critique-and-gaps.md M1 + owner_step: deferred + status: deferred + decision_gated: >- + Whether the DAST confirmation model must be the disputed 9B pick or can be smaller. PASS (a + 2-4B model matches the 9B pick under grammar-constrained tool calling): use the smaller model. + FAIL: keep the DAST model at the same size as the SAST model and scale turn budget and context + instead of parameters — research/14 M1's stated fix. + corpus: [] + method: >- + Head-to-head DAST confirmation eval of a 2-4B model against the 9B pick under + grammar-constrained, template-bounded tool calling. + metric: Confirmation accuracy of the 2-4B arm relative to the 9B arm under identical constraints. + threshold_pass: >- + A 2-4B model matches the 9B pick under grammar-constrained tool calling. PROPOSED, informed by + research/14-critique-and-gaps.md M1's "same-size-or-smaller" verdict — 5 branches argue against + the 9B pick, 2 of them survivor-biased or self-reported. + threshold_fail: >- + The 2-4B arm does not match the 9B arm; keep parity with the SAST model size and scale turn + budget / context instead of parameters (research/14 M1's fix). + reference_baseline: PROPOSED — no published baseline + result: + value: null + unit: null + measured_at: null + artifact_path: null + decision: UNRESOLVED + depends_on: [] + deferred_reason: >- + Milestone 0 executes only the six items named in plan/10-milestone0-evaluation.md "## Overview", + and no DAST confirmation loop exists to evaluate. Per plan/00-SPINE.md S9-AMENDED, DAST is a + separate distribution artifact whose model selection is not an M0 decision. + owning_future_phase: null + + # ========================================================================= + - id: EXP-09 + name: runsc vs. runc overhead + kind: experiment + spine_ref: S9 + source_ref: >- + research/19-target-environment-and-sandboxing.md; enumerated as experiment #9 in + research/14-critique-and-gaps.md B7 table ("whether the sandbox fits the budget") + owner_step: deferred + status: deferred + decision_gated: >- + Whether gVisor's isolation is affordable at the declared hardware budget. PASS: gVisor overhead + fits within Tier-M's 32 GB / 8-core budget (plan/00-SPINE.md S9). FAIL: narrow gVisor to only + the highest-risk boundary, or raise the DAST-enabled hardware floor. + corpus: [] + method: >- + Run the actual probe workload under runsc and under runc and compare CPU, memory and wall-clock + overhead. + metric: runsc-vs-runc overhead (CPU, RSS, wall clock) on the actual probe workload. + threshold_pass: >- + Measured overhead fits within Tier-M's 32 GB / 8-core budget (plan/00-SPINE.md S9). PROPOSED — + no published Anvil-specific number. + threshold_fail: >- + Measured overhead exceeds the Tier-M budget. PROPOSED — no published Anvil-specific number. + reference_baseline: PROPOSED — no published baseline + result: + value: null + unit: null + measured_at: null + artifact_path: null + decision: UNRESOLVED + depends_on: [] + deferred_reason: >- + Milestone 0 executes only the six items named in plan/10-milestone0-evaluation.md "## Overview". + There is no probe workload to sandbox at M0; per plan/00-SPINE.md S9-AMENDED the DAST sandbox + belongs to a separate distribution artifact. + owning_future_phase: null + + # ========================================================================= + - id: EXP-10 + name: ZAP JVM RSS under a representative full scan + kind: experiment + spine_ref: S9 + source_ref: >- + research/15-dast-tooling-landscape.md; enumerated as experiment #10 in + research/14-critique-and-gaps.md B7 table ("whether ZAP can be always-on"); footprint recorded + as unquantified in research/14-critique-and-gaps.md C12 + owner_step: deferred + status: deferred + decision_gated: >- + Whether ZAP can be always-on. PASS: ZAP's measured RSS fits the opt-in DAST tier budget + (plan/00-SPINE.md S9). FAIL: confirms S9's existing default ("do not run ZAP always-on") — no + redesign needed, just confirmation. This is the one row where a FAIL changes nothing. + corpus: [] + method: >- + Measure ZAP JVM resident set size during a representative full scan; the proposed method is a + 10-minute docker stats observation (research/14-critique-and-gaps.md C12). + metric: Peak and steady-state JVM RSS (MB/GB) during a representative full scan. + threshold_pass: >- + Measured RSS fits the opt-in DAST tier budget (plan/00-SPINE.md S9). PROPOSED — research/14 + C12 records the ZAP footprint as "unquantified", so there is no published figure to compare to. + threshold_fail: >- + Measured RSS exceeds the opt-in DAST tier budget — which merely confirms S9's existing + always-on default. + reference_baseline: PROPOSED — no published baseline + result: + value: null + unit: null + measured_at: null + artifact_path: null + decision: UNRESOLVED + depends_on: [] + deferred_reason: >- + Milestone 0 executes only the six items named in plan/10-milestone0-evaluation.md "## Overview". + No ZAP deployment exists at M0, and per plan/00-SPINE.md S9-AMENDED DAST ships as a separate + distribution artifact whose resource budget is not an M0 gate. + owning_future_phase: null + + # ========================================================================= + - id: EXP-11 + name: Task cards vs. raw SARIF + kind: experiment + spine_ref: S6 + source_ref: >- + research/18-unified-audit-record.md — branch 18's "the single highest-value thing to A/B test + early"; the eleventh experiment in research/14-critique-and-gaps.md B7 + owner_step: deferred + status: deferred + decision_gated: >- + Whether the task-card abstraction earns its own schema slot in plan/00-SPINE.md S6. PASS: it + does. FAIL: feed raw SARIF excerpts directly to the coding agent and drop the task-card + abstraction layer entirely. + corpus: + - name: ARVO (subset reused from EXP-04) + licence: BSD-2-Clause + licence_ref: research/16-fuzzing-and-dynamic-analysis.md S2/S25 + - name: CWE-Bench-Java (subset reused from EXP-04) + licence: MIT + licence_ref: plan/spine-b-open-licences.md section 7 + method: >- + A/B compare coding-agent fix-success rate given task cards versus raw SARIF excerpts, reusing + EXP-04's ARVO / CWE-Bench-Java subset so the two arms differ only in prompt representation. + metric: Coding-agent fix-success rate, task-card arm vs. raw-SARIF arm, on the identical subset. + threshold_pass: >- + The task-card arm's fix-success rate materially exceeds the raw-SARIF arm's. PROPOSED — no + published baseline; the A/B design is specified but the margin is not. + threshold_fail: >- + The task-card arm does not beat raw SARIF excerpts. PROPOSED — no published baseline. + reference_baseline: PROPOSED — no published baseline + result: + value: null + unit: null + measured_at: null + artifact_path: null + decision: UNRESOLVED + depends_on: + - EXP-04 + deferred_reason: >- + Depends on EXP-04's corpus subset and harness, which do not exist until M0.16 runs, and on + whether plan/00-SPINE.md S6 gains a task-card schema slot at all. Milestone 0 executes only the + six items named in plan/10-milestone0-evaluation.md "## Overview". + owning_future_phase: null + + # ========================================================================= + - id: EXP-12 + name: llama-server LoRA hot-swap + kind: experiment + spine_ref: S3, S4 + source_ref: >- + research/05-inference-serving-and-hardware.md — "potentially the highest-leverage unexplored + option"; the twelfth experiment in research/14-critique-and-gaps.md B7 + owner_step: deferred + status: deferred + decision_gated: >- + Only relevant if a later milestone trains adapters: hot-swap latency versus static loading, and + therefore whether multi-adapter serving is viable. FAIL / not-applicable while v1 ships + zero-training (research/03's primary recommendation). Note plan/00-SPINE.md S4: "Not vLLM in + v1 ... requires fine-tuned adapters that must not exist before S3." + corpus: [] + method: >- + Measure llama-server LoRA hot-swap latency against static adapter loading, once adapters exist. + metric: Hot-swap latency (ms) vs. static-load latency (ms). + threshold_pass: >- + PROPOSED — entirely contingent on EXP-01/EXP-02 justifying training adapters at all; no bar can + be set before that determination. + threshold_fail: >- + PROPOSED — no bar can be set before EXP-01/EXP-02 resolve. This row is deferred, never "not + applicable": if the detection tier survives and a later milestone trains adapters, it becomes + live. + reference_baseline: PROPOSED — no published baseline + result: + value: null + unit: null + measured_at: null + artifact_path: null + decision: UNRESOLVED + depends_on: + - EXP-01 + - EXP-02 + deferred_reason: >- + Entirely contingent on EXP-01 and EXP-02 justifying training adapters at all + (plan/10-milestone0-evaluation.md "## Go/No-Go Decision Table", EXP-12 row), and both of those + rows are UNRESOLVED. v1 ships zero-training (research/03 primary recommendation), and + plan/00-SPINE.md S3/S4 forbids fine-tuned adapters existing before S3 resolves, so there is + nothing to hot-swap. + owning_future_phase: null diff --git a/eval/requirements.txt b/eval/requirements.txt new file mode 100644 index 0000000..f4d57db --- /dev/null +++ b/eval/requirements.txt @@ -0,0 +1,45 @@ +# Anvil evaluation harness — resolved pin set (M0.2) +# +# pyproject.toml holds the *declared* dependency floors; this file holds the +# *resolved* versions, so an eval run can be reproduced exactly. Generated by +# `pip freeze --exclude-editable` in a clean virtualenv on 2026-08-06 with +# CPython 3.13.5 / pip 26.2.1 / setuptools 83.0.0 on win_amd64. +# +# Covers the core install plus the [dev] extra (pytest + ruff). The [stats] +# and [models] extras are intentionally NOT pinned here: they are large, +# platform-specific, and only M0.6/M0.10/M0.15 need them. Those steps pin +# their own artifacts (S8: exact revision SHAs). +# +# Reproduce with: +# python -m venv .venv +# .venv/Scripts/python -m pip install -r eval/requirements.txt +# .venv/Scripts/python -m pip install -e eval/ --no-deps + +# --- core (project.dependencies) ------------------------------------------- +jsonschema==4.26.0 +numpy==2.5.1 +PyYAML==6.0.3 +requests==2.34.2 + +# --- transitive closure of the core set ------------------------------------ +attrs==26.1.0 +certifi==2026.7.22 +charset-normalizer==3.4.9 +idna==3.18 +jsonschema-specifications==2025.9.1 +referencing==0.37.0 +rpds-py==2026.6.3 +urllib3==2.7.0 + +# --- [dev] extra ------------------------------------------------------------ +pytest==9.1.1 +pytest-cov==7.1.0 +ruff==0.16.1 + +# --- transitive closure of [dev] ------------------------------------------- +colorama==0.4.6 +coverage==7.15.4 +iniconfig==2.3.0 +packaging==26.3 +pluggy==1.6.0 +Pygments==2.20.0 diff --git a/eval/schema/register.schema.json b/eval/schema/register.schema.json new file mode 100644 index 0000000..21cf990 --- /dev/null +++ b/eval/schema/register.schema.json @@ -0,0 +1,225 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://anvil.invalid/eval/schema/register.schema.json", + "title": "Anvil Milestone 0 Experiment Register", + "description": "Schema for eval/register.yaml. Source of truth for the field list: plan/10-milestone0-evaluation.md '## Experiment Register Schema'. The register carries exactly fourteen rows: EXP-01..EXP-12, INSTR-01, S12-RTT. SAFETY RULE: 'decision' is required on every row and has no null / no empty-string / no absent representation. An experiment whose outcome has not been adjudicated carries the explicit sentinel 'UNRESOLVED'. A missing or unadjudicated decision must never be readable as a PASS.", + "type": "object", + "additionalProperties": false, + "required": ["version", "experiments"], + "properties": { + "version": { + "description": "Register format version. 1 per plan/10-milestone0-evaluation.md.", + "type": "integer", + "minimum": 1 + }, + "experiments": { + "description": "Exactly the fourteen registered rows, each ID present exactly once.", + "type": "array", + "minItems": 14, + "maxItems": 14, + "items": { "$ref": "#/$defs/row" }, + "allOf": [ + { "contains": { "$ref": "#/$defs/idIs/EXP-01" }, "minContains": 1, "maxContains": 1 }, + { "contains": { "$ref": "#/$defs/idIs/EXP-02" }, "minContains": 1, "maxContains": 1 }, + { "contains": { "$ref": "#/$defs/idIs/EXP-03" }, "minContains": 1, "maxContains": 1 }, + { "contains": { "$ref": "#/$defs/idIs/EXP-04" }, "minContains": 1, "maxContains": 1 }, + { "contains": { "$ref": "#/$defs/idIs/EXP-05" }, "minContains": 1, "maxContains": 1 }, + { "contains": { "$ref": "#/$defs/idIs/EXP-06" }, "minContains": 1, "maxContains": 1 }, + { "contains": { "$ref": "#/$defs/idIs/EXP-07" }, "minContains": 1, "maxContains": 1 }, + { "contains": { "$ref": "#/$defs/idIs/EXP-08" }, "minContains": 1, "maxContains": 1 }, + { "contains": { "$ref": "#/$defs/idIs/EXP-09" }, "minContains": 1, "maxContains": 1 }, + { "contains": { "$ref": "#/$defs/idIs/EXP-10" }, "minContains": 1, "maxContains": 1 }, + { "contains": { "$ref": "#/$defs/idIs/EXP-11" }, "minContains": 1, "maxContains": 1 }, + { "contains": { "$ref": "#/$defs/idIs/EXP-12" }, "minContains": 1, "maxContains": 1 }, + { "contains": { "$ref": "#/$defs/idIs/INSTR-01" }, "minContains": 1, "maxContains": 1 }, + { "contains": { "$ref": "#/$defs/idIs/S12-RTT" }, "minContains": 1, "maxContains": 1 } + ] + } + }, + "$defs": { + "idIs": { + "EXP-01": { "type": "object", "properties": { "id": { "const": "EXP-01" } }, "required": ["id"] }, + "EXP-02": { "type": "object", "properties": { "id": { "const": "EXP-02" } }, "required": ["id"] }, + "EXP-03": { "type": "object", "properties": { "id": { "const": "EXP-03" } }, "required": ["id"] }, + "EXP-04": { "type": "object", "properties": { "id": { "const": "EXP-04" } }, "required": ["id"] }, + "EXP-05": { "type": "object", "properties": { "id": { "const": "EXP-05" } }, "required": ["id"] }, + "EXP-06": { "type": "object", "properties": { "id": { "const": "EXP-06" } }, "required": ["id"] }, + "EXP-07": { "type": "object", "properties": { "id": { "const": "EXP-07" } }, "required": ["id"] }, + "EXP-08": { "type": "object", "properties": { "id": { "const": "EXP-08" } }, "required": ["id"] }, + "EXP-09": { "type": "object", "properties": { "id": { "const": "EXP-09" } }, "required": ["id"] }, + "EXP-10": { "type": "object", "properties": { "id": { "const": "EXP-10" } }, "required": ["id"] }, + "EXP-11": { "type": "object", "properties": { "id": { "const": "EXP-11" } }, "required": ["id"] }, + "EXP-12": { "type": "object", "properties": { "id": { "const": "EXP-12" } }, "required": ["id"] }, + "INSTR-01": { "type": "object", "properties": { "id": { "const": "INSTR-01" } }, "required": ["id"] }, + "S12-RTT": { "type": "object", "properties": { "id": { "const": "S12-RTT" } }, "required": ["id"] } + }, + + "row": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "name", + "kind", + "spine_ref", + "source_ref", + "owner_step", + "status", + "decision_gated", + "corpus", + "method", + "metric", + "threshold_pass", + "threshold_fail", + "reference_baseline", + "result", + "decision", + "depends_on", + "deferred_reason", + "owning_future_phase" + ], + "properties": { + "id": { + "description": "Stable identifier, never reused.", + "type": "string", + "enum": [ + "EXP-01", "EXP-02", "EXP-03", "EXP-04", "EXP-05", "EXP-06", + "EXP-07", "EXP-08", "EXP-09", "EXP-10", "EXP-11", "EXP-12", + "INSTR-01", "S12-RTT" + ] + }, + "name": { "type": "string", "minLength": 1 }, + "kind": { "type": "string", "enum": ["experiment", "instrumentation"] }, + "spine_ref": { + "description": "Section(s) of plan/00-SPINE.md this row is gated on, e.g. 'S3.1', 'S2', 'S12'.", + "type": "string", + "minLength": 1 + }, + "source_ref": { + "description": "Research file + section that named this experiment.", + "type": "string", + "minLength": 1 + }, + "owner_step": { + "description": "The M0.x step ID that builds/runs it, or the literal 'deferred'.", + "type": "string", + "pattern": "^(M0\\.[0-9]+|deferred)$" + }, + "status": { + "type": "string", + "enum": ["not_started", "in_progress", "blocked", "complete", "deferred"] + }, + "decision_gated": { + "description": "Free text: what this outcome decides.", + "type": "string", + "minLength": 1 + }, + "corpus": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name", "licence", "licence_ref"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "licence": { "type": "string", "minLength": 1 }, + "licence_ref": { + "description": "File + section confirming the licence.", + "type": "string", + "minLength": 1 + } + } + } + }, + "method": { "type": "string", "minLength": 1 }, + "metric": { "type": "string", "minLength": 1 }, + "threshold_pass": { "type": "string", "minLength": 1 }, + "threshold_fail": { "type": "string", "minLength": 1 }, + "reference_baseline": { + "description": "Per plan/10-milestone0-evaluation.md '## Exit Criteria': must either cite a research/plan file + figure (i.e. contain a '.md' path) or be the literal string 'PROPOSED - no published baseline' (em dash).", + "type": "string", + "anyOf": [ + { "const": "PROPOSED — no published baseline" }, + { "pattern": "\\.md" } + ] + }, + "result": { + "type": "object", + "additionalProperties": false, + "required": ["value", "unit", "measured_at", "artifact_path"], + "properties": { + "value": { "type": ["number", "null"] }, + "unit": { "type": ["string", "null"] }, + "measured_at": { + "type": ["string", "null"], + "description": "ISO-8601 date or date-time; null until measured." + }, + "artifact_path": { + "type": ["string", "null"], + "description": "eval/results/.json; null until produced." + } + } + }, + "decision": { + "description": "The adjudicated outcome. REQUIRED on every row. There is deliberately no null, no empty string and no absent representation: an unadjudicated row carries the explicit sentinel 'UNRESOLVED', which must never be read as a PASS. Only an orchestrator-inline step may change this away from 'UNRESOLVED' (M0.18 for EXP-01/EXP-02; future orchestrator-inline steps for the rest). Worker packets populate 'result' and leave 'decision' at 'UNRESOLVED'.", + "type": "string", + "enum": ["PASS", "FAIL", "AMBIGUOUS", "DEFERRED", "UNRESOLVED"], + "default": "UNRESOLVED" + }, + "depends_on": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "EXP-01", "EXP-02", "EXP-03", "EXP-04", "EXP-05", "EXP-06", + "EXP-07", "EXP-08", "EXP-09", "EXP-10", "EXP-11", "EXP-12", + "INSTR-01", "S12-RTT" + ] + }, + "uniqueItems": true + }, + "deferred_reason": { "type": ["string", "null"] }, + "owning_future_phase": { + "description": "Milestone/lane that will own this row once deferred. Populated by step M0.17; null until then. Milestone-0 exit criteria require this to be non-null for every deferred row before M0 closes.", + "type": ["string", "null"] + } + }, + "allOf": [ + { + "description": "A deferred row must carry a non-empty deferred_reason and must be owned by 'deferred', not an M0 step.", + "if": { "properties": { "status": { "const": "deferred" } }, "required": ["status"] }, + "then": { + "properties": { + "deferred_reason": { "type": "string", "minLength": 1 }, + "owner_step": { "const": "deferred" } + } + } + }, + { + "description": "A row claiming status 'complete' must carry a measured value and a real artifact path.", + "if": { "properties": { "status": { "const": "complete" } }, "required": ["status"] }, + "then": { + "properties": { + "result": { + "properties": { + "value": { "type": "number" }, + "unit": { "type": "string", "minLength": 1 }, + "measured_at": { "type": "string", "minLength": 1 }, + "artifact_path": { "type": "string", "pattern": "^eval/results/.+\\.json$" } + } + } + } + } + }, + { + "description": "A row that has not been run cannot carry an adjudicated decision. not_started / in_progress / blocked / deferred rows stay UNRESOLVED. 'deferred' is in this set deliberately: a deferred row has by definition not been run, so it cannot have an adjudicated outcome. Omitting it was caught by mutation testing -- EXP-05 (deferred) could be set to PASS and the schema accepted it, which is exactly the 'a missing decision reads as a pass' failure this rule exists to prevent. A row that is deferred AND wants to record that fact in the decision field uses status=deferred + decision=UNRESOLVED; the DEFERRED decision value is reserved for an orchestrator-inline step explicitly adjudicating 'we decided not to decide this'.", + "if": { + "properties": { "status": { "enum": ["not_started", "in_progress", "blocked", "deferred"] } }, + "required": ["status"] + }, + "then": { "properties": { "decision": { "const": "UNRESOLVED" } } } + } + ] + } + } +} diff --git a/eval/src/anvil_eval/__init__.py b/eval/src/anvil_eval/__init__.py new file mode 100644 index 0000000..750576b --- /dev/null +++ b/eval/src/anvil_eval/__init__.py @@ -0,0 +1,80 @@ +"""Anvil Milestone 0 evaluation harness. + +This package is a **pure-Python** tree. It exists under the third of the three +carve-outs in ``plan/00-SPINE.md`` S12 ("Where Python survives — exactly three +places, none of them control-plane runtime"): *the evaluation harness and +KL-divergence quantisation checks*. No Go code belongs here, and nothing in this +tree is part of the Anvil control plane. + +The package provides the shared skeleton that every later Milestone 0 step +builds on: + +* ``anvil_eval.data`` — corpus loaders (M0.3 PrimeVul, M0.4 ARVO, M0.5 CWE-Bench-Java) +* ``anvil_eval.harness`` — experiment runners (M0.9…M0.16) + +Those submodules are written by later packets; this module only fixes the +version, the on-disk layout, and the small helpers that keep every step +pointing at the same directories. +""" + +from __future__ import annotations + +from pathlib import Path + +__all__ = [ + "__version__", + "EVAL_ROOT", + "REPO_ROOT", + "DATA_DIR", + "MODELS_DIR", + "TOOLS_DIR", + "RESULTS_DIR", + "NOTES_DIR", + "HARNESS_DIR", + "SCHEMA_DIR", + "REGISTER_PATH", + "REGISTER_SCHEMA_PATH", + "result_path", +] + +__version__ = "0.1.0" + +#: Root of the ``eval/`` tree. Resolved from this file's location so it is +#: correct for an editable install (``pip install -e eval/``) regardless of the +#: process working directory. +EVAL_ROOT: Path = Path(__file__).resolve().parents[2] + +#: Repository root (the parent of ``eval/``). +REPO_ROOT: Path = EVAL_ROOT.parent + +# Canonical sub-trees. These paths are the contract between M0 steps; a step +# that writes somewhere else breaks the register's ``artifact_path`` fields. +DATA_DIR: Path = EVAL_ROOT / "data" # M0.3, M0.4, M0.5 — gitignored payloads +MODELS_DIR: Path = EVAL_ROOT / "models" # M0.6 — pinned-download manifests, not weights +TOOLS_DIR: Path = EVAL_ROOT / "tools" # M0.7 — opengrep engine + pinned ruleset +RESULTS_DIR: Path = EVAL_ROOT / "results" # eval/results/.json, committed +NOTES_DIR: Path = EVAL_ROOT / "notes" # licence findings and critic verdicts +HARNESS_DIR: Path = EVAL_ROOT / "harness" # experiment runner scripts +SCHEMA_DIR: Path = EVAL_ROOT / "schema" # M0.1 — register JSON Schema + +#: The experiment register (authored by M0.1, updated by M0.17/M0.18). +REGISTER_PATH: Path = EVAL_ROOT / "register.yaml" + +#: JSON Schema the register validates against (authored by M0.1). +REGISTER_SCHEMA_PATH: Path = SCHEMA_DIR / "register.schema.json" + + +def result_path(experiment_id: str) -> Path: + """Return the canonical result artifact path for an experiment ID. + + The register's ``result.artifact_path`` field is specified as + ``eval/results/.json`` in ``plan/10-milestone0-evaluation.md``; every + experiment step must write there and nowhere else. + + >>> result_path("EXP-01").name + 'EXP-01.json' + """ + experiment_id = experiment_id.strip() + if not experiment_id: + raise ValueError("experiment_id must be a non-empty string") + return RESULTS_DIR / f"{experiment_id}.json" diff --git a/eval/tests/test_dependency_manifest.py b/eval/tests/test_dependency_manifest.py new file mode 100644 index 0000000..711a54e --- /dev/null +++ b/eval/tests/test_dependency_manifest.py @@ -0,0 +1,73 @@ +"""Tests over the M0.2 dependency manifest itself. + +These guard the two constraints the M0.2 packet names explicitly: +no cgo/Go SQLite dependency has any business in this tree, and no dataset or +model weight is vendored by the scaffold. +""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +import pytest + +EVAL_ROOT = Path(__file__).resolve().parents[1] +PYPROJECT = EVAL_ROOT / "pyproject.toml" + + +@pytest.fixture(scope="module") +def pyproject() -> dict: + with PYPROJECT.open("rb") as fh: + return tomllib.load(fh) + + +def test_pyproject_is_valid_toml_and_names_the_package(pyproject: dict) -> None: + assert pyproject["project"]["name"] == "anvil-eval" + assert pyproject["project"]["requires-python"].startswith(">=3.") + + +def test_src_layout_is_declared(pyproject: dict) -> None: + assert pyproject["tool"]["setuptools"]["packages"]["find"]["where"] == ["src"] + assert (EVAL_ROOT / "src" / "anvil_eval" / "__init__.py").is_file() + + +def test_no_go_or_cgo_flavoured_dependency_is_declared(pyproject: dict) -> None: + """Forbidden action (M0.2): no ``mattn/go-sqlite3``, no cgo dependency.""" + declared = list(pyproject["project"]["dependencies"]) + for group in pyproject["project"].get("optional-dependencies", {}).values(): + declared.extend(group) + lowered = " ".join(declared).lower() + for banned in ("go-sqlite3", "mattn", "cgo"): + assert banned not in lowered, f"forbidden dependency token {banned!r} in manifest" + + +def test_requirements_file_exists_and_is_pinned() -> None: + """`requirements.txt` is the reproducible pin set for the core+dev install.""" + requirements = EVAL_ROOT / "requirements.txt" + assert requirements.is_file() + pins = [ + line.strip() + for line in requirements.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + assert pins, "requirements.txt declares no pins" + for pin in pins: + assert "==" in pin, f"unpinned requirement: {pin!r}" + + +def test_scaffold_vendors_no_dataset_or_model_weight() -> None: + """Forbidden action (M0.2): do not vendor any dataset or model weight. + + Scoped to the paths M0.2 owns — ``eval/data/`` and ``eval/models/`` are + later steps' write scope and are gitignored payload directories. + """ + weight_suffixes = {".gguf", ".safetensors", ".onnx", ".bin", ".pt", ".pth", ".h5", ".ckpt"} + scanned = [EVAL_ROOT / "src", EVAL_ROOT / "tests"] + offenders = [ + p + for root in scanned + for p in root.rglob("*") + if p.is_file() and p.suffix.lower() in weight_suffixes + ] + assert offenders == [], f"model/dataset payloads vendored under eval/: {offenders}" diff --git a/eval/tests/test_register_schema.py b/eval/tests/test_register_schema.py new file mode 100644 index 0000000..4b66a6c --- /dev/null +++ b/eval/tests/test_register_schema.py @@ -0,0 +1,146 @@ +"""The experiment register must validate, and an unrun row must never read as a pass. + +plan step M0.18 reads eval/register.yaml and decides whether Anvil's detection-model +tier exists at all. Two rows -- EXP-01 (advisory-permutation ablation) and EXP-02 +(code-metrics baseline) -- can delete that tier entirely. + +That makes exactly one property load-bearing: **a row whose experiment has not been +run must be impossible to confuse with a row that passed.** Everything else in the +register is bookkeeping; this is the safety property. + +These tests exist because the register and its schema shipped with nothing anywhere +in the repository actually checking one against the other -- the validation was run +once, by hand, at authoring time. A schema nobody runs is a comment. + +The negative cases are mutation tests. Each one takes the real register, introduces +one specific way an undecided row could be made to read as decided, and asserts the +schema rejects it. The `deferred` case is here because it was a real bug: the guard +originally covered not_started/in_progress/blocked and omitted deferred, so EXP-05 +could be set to PASS and validation accepted it. +""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import jsonschema +import pytest +import yaml + +EVAL_ROOT = Path(__file__).resolve().parent.parent +SCHEMA_PATH = EVAL_ROOT / "schema" / "register.schema.json" +REGISTER_PATH = EVAL_ROOT / "register.yaml" + +# The fourteen rows the plan requires. Named explicitly rather than counted, so a +# row being renamed fails loudly instead of silently keeping the count right. +REQUIRED_IDS = [ + *(f"EXP-{n:02d}" for n in range(1, 13)), + "INSTR-01", + "S12-RTT", +] + +# Any status meaning "this experiment has not produced an adjudicated outcome". +UNRUN_STATUSES = ["not_started", "in_progress", "blocked", "deferred"] + + +@pytest.fixture(scope="module") +def schema() -> dict: + return json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + + +@pytest.fixture(scope="module") +def register() -> dict: + return yaml.safe_load(REGISTER_PATH.read_text(encoding="utf-8")) + + +def _rows(register: dict) -> list[dict]: + """The register may carry its rows under a key or as a bare list.""" + if isinstance(register, dict): + for key in ("experiments", "rows", "register"): + if isinstance(register.get(key), list): + return register[key] + raise AssertionError(f"no row list found in register keys: {sorted(register)}") + return register + + +def _errors(schema: dict, doc: dict) -> list: + return list(jsonschema.Draft202012Validator(schema).iter_errors(doc)) + + +def test_schema_is_itself_valid(schema: dict) -> None: + jsonschema.Draft202012Validator.check_schema(schema) + + +def test_register_validates(schema: dict, register: dict) -> None: + errors = _errors(schema, register) + assert not errors, "register.yaml does not validate:\n" + "\n".join( + f" {list(e.absolute_path)}: {e.message}" for e in errors + ) + + +def test_all_fourteen_rows_present_exactly_once(register: dict) -> None: + ids = [r["id"] for r in _rows(register)] + assert sorted(ids) == sorted(REQUIRED_IDS), ( + f"missing={sorted(set(REQUIRED_IDS) - set(ids))} " + f"unexpected={sorted(set(ids) - set(REQUIRED_IDS))}" + ) + assert len(ids) == len(set(ids)), "duplicate row ids" + + +def test_no_row_has_a_null_absent_or_empty_decision(register: dict) -> None: + """The sentinel must be explicit. Absence is the failure mode being prevented.""" + for row in _rows(register): + assert "decision" in row, f"{row['id']}: decision key absent" + assert row["decision"] not in (None, ""), f"{row['id']}: decision is null/empty" + + +def test_every_unrun_row_is_unresolved(register: dict) -> None: + for row in _rows(register): + if row["status"] in UNRUN_STATUSES: + assert row["decision"] == "UNRESOLVED", ( + f"{row['id']} has status={row['status']} but decision={row['decision']!r}. " + "An experiment that has not run cannot carry an adjudicated outcome." + ) + + +@pytest.mark.parametrize("status", UNRUN_STATUSES) +def test_schema_rejects_a_pass_on_an_unrun_row(schema: dict, register: dict, status: str) -> None: + """Mutation test, one per unrun status. + + This is the test that would have caught the `deferred` hole: the guard covered + three of the four unrun statuses, so a deferred row could be marked PASS. + """ + mutated = copy.deepcopy(register) + rows = _rows(mutated) + rows[0]["status"] = status + rows[0]["decision"] = "PASS" + # A deferred row carries its own extra obligations; satisfy them so the only + # reason validation can fail is the decision itself. + if status == "deferred": + rows[0]["deferred_reason"] = "mutation test" + rows[0]["owner_step"] = "deferred" + + assert _errors(schema, mutated), ( + f"schema ACCEPTED decision=PASS on a status={status} row. " + "An unadjudicated row can be made to read as a pass." + ) + + +@pytest.mark.parametrize( + "field,value", + [("decision", None), ("decision", ""), ("decision", "OK")], +) +def test_schema_rejects_malformed_decisions( + schema: dict, register: dict, field: str, value +) -> None: + mutated = copy.deepcopy(register) + _rows(mutated)[0][field] = value + assert _errors(schema, mutated), f"schema accepted {field}={value!r}" + + +def test_schema_rejects_a_deleted_row(schema: dict, register: dict) -> None: + mutated = copy.deepcopy(register) + del _rows(mutated)[0] + assert _errors(schema, mutated), "schema accepted a register with a row removed" diff --git a/eval/tests/test_scaffold.py b/eval/tests/test_scaffold.py new file mode 100644 index 0000000..e15eb72 --- /dev/null +++ b/eval/tests/test_scaffold.py @@ -0,0 +1,90 @@ +"""Scaffold tests for the Anvil evaluation harness (M0.2). + +These are deliberately offline and dependency-light: they assert that the +package imports, that its declared on-disk layout is self-consistent, and that +the S12 carve-out is honoured (``eval/`` is a pure-Python tree — no Go sources). +Later M0 packets add their own test modules alongside this one. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import anvil_eval + +#: Paths this packet (M0.2) owns. Scans below are scoped to these so that they +#: stay fast and do not fail on artifacts legitimately produced by later steps +#: (e.g. M0.6's model downloads under ``eval/models/``). +SCAFFOLD_PATHS = ("pyproject.toml", "requirements.txt", ".gitignore", "src", "tests") + + +def test_package_imports_and_declares_a_version() -> None: + assert isinstance(anvil_eval.__version__, str) + assert anvil_eval.__version__.count(".") == 2, "expected a MAJOR.MINOR.PATCH version" + + +def test_eval_root_resolves_to_this_checkout() -> None: + """EVAL_ROOT must point at the ``eval/`` tree containing this test file.""" + expected = Path(__file__).resolve().parents[1] + assert anvil_eval.EVAL_ROOT == expected + assert (anvil_eval.EVAL_ROOT / "pyproject.toml").is_file() + + +def test_declared_subtrees_live_under_eval_root() -> None: + """Every canonical directory constant must be inside ``eval/``. + + Later steps write results and manifests through these constants; if one of + them ever escaped the eval tree it would write into the Go control plane. + """ + subtrees = [ + anvil_eval.DATA_DIR, + anvil_eval.MODELS_DIR, + anvil_eval.TOOLS_DIR, + anvil_eval.RESULTS_DIR, + anvil_eval.NOTES_DIR, + anvil_eval.HARNESS_DIR, + anvil_eval.SCHEMA_DIR, + anvil_eval.REGISTER_PATH, + anvil_eval.REGISTER_SCHEMA_PATH, + ] + for path in subtrees: + assert path.is_relative_to(anvil_eval.EVAL_ROOT), f"{path} escapes EVAL_ROOT" + + +def test_result_path_matches_the_registers_artifact_path_convention() -> None: + """`plan/10-milestone0-evaluation.md` specifies ``eval/results/.json``.""" + for experiment_id in ("EXP-01", "EXP-12", "INSTR-01", "S12-RTT"): + path = anvil_eval.result_path(experiment_id) + assert path.parent == anvil_eval.RESULTS_DIR + assert path.name == f"{experiment_id}.json" + + +def test_result_path_rejects_an_empty_id() -> None: + with pytest.raises(ValueError): + anvil_eval.result_path(" ") + + +def _scaffold_files() -> list[Path]: + files: list[Path] = [] + for name in SCAFFOLD_PATHS: + target = anvil_eval.EVAL_ROOT / name + if target.is_file(): + files.append(target) + elif target.is_dir(): + files.extend(p for p in target.rglob("*") if p.is_file()) + return files + + +def test_scaffold_contains_no_go_sources() -> None: + """S12 carve-out #3: the evaluation harness is a pure-Python tree. + + The M0.2 packet is explicit that "no Go code belongs in ``eval/`` at all". + """ + offenders = [ + p + for p in _scaffold_files() + if p.suffix == ".go" or p.name in {"go.mod", "go.sum"} + ] + assert offenders == [], f"Go artifacts found under eval/: {offenders}" diff --git a/eval/tools/opengrep/.gitignore b/eval/tools/opengrep/.gitignore new file mode 100644 index 0000000..e4883e2 --- /dev/null +++ b/eval/tools/opengrep/.gitignore @@ -0,0 +1,12 @@ +# Acquired artefacts: fetched from pinned URLs, verified against MANIFEST.toml, +# never committed. The pin lives in MANIFEST.toml; the bytes do not live in git. +vendor/ + +# opengrep binaries are LGPL-2.1. Keeping them out of the tree also keeps Anvil +# out of any "distribution" question until plan/30-lane-b-detection.md B.2 +# decides packaging deliberately. +opengrep +opengrep.exe +opengrep_* +opengrep-core_* +*.sarif diff --git a/eval/tools/opengrep/LICENSES.md b/eval/tools/opengrep/LICENSES.md new file mode 100644 index 0000000..f51ae17 --- /dev/null +++ b/eval/tools/opengrep/LICENSES.md @@ -0,0 +1,161 @@ +# M0.7 — licence findings for the deterministic recall tier + +Every claim below was fetched in-session on **2026-08-06** from a primary source and is quoted from +the LICENSE file body, not from an API `license` metadata field. Where an API field is cited it is +cited as corroboration only, never as the finding. This follows `plan/00-SPINE.md` S8's compliance +mechanic ("reads LICENSE file bodies, never API metadata") and the verification discipline in +`research/13-license-compatibility-audit.md` / `research/14-critique-and-gaps.md`. + +--- + +## 1. Engine — `opengrep/opengrep` @ `v1.26.0` — **LGPL-2.1** + +**Fetched:** `https://raw.githubusercontent.com/opengrep/opengrep/v1.26.0/LICENSE` +(504 lines; the pin is the tag, so the fetch is reproducible.) + +Opening lines of the LICENSE body, verbatim: + +``` + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. +``` + +**Finding: LGPL-2.1.** Consistent with `plan/00-SPINE.md` S4 and with +`research/10-prior-art-and-landscape.md` [S11] ("`opengrep/opengrep` is **LGPL-2.1**, ~9,931 commits, +forked from Semgrep at version 1.100.0"). + +**Release pin (GitHub Releases API, `/repos/opengrep/opengrep/releases/latest`):** + +| Field | Value | +|---|---| +| Tag | `v1.26.0` | +| Tagged commit | `1bef4ea4ff3264754132eec823b5b1d8cde3e4ee` | +| Published | 2026-07-24T20:00:53Z | +| Prerelease | false | + +### Linkage posture — this is the whole compliance argument, and it is short + +opengrep is an **OCaml CLI with zero bindings in any language** (`plan/00-SPINE.md` S12). There is no +FFI surface to link against even if someone wanted to. Anvil therefore `exec`s the binary and reads +its stdout. No cgo, no shared object, no static link, no in-process plugin. LGPL-2.1's combined-work +obligations (§6) are about works that *link* the library; a program that shells out to a separate +executable and talks to it over a pipe is not one. `plan/30-lane-b-detection.md` B.1 makes the +subprocess-only rule a build-enforced invariant on the Go side; this evaluation harness holds the +same line by construction — `anvil_opengrep/runner.py` only ever calls `subprocess.run`. + +### The obligation that *does* attach, and is deferred, not dismissed + +If Anvil ever **distributes** the compiled opengrep binary — e.g. bakes it into a published container +image — that is conveyance of an LGPL-2.1 work and triggers notice + offer-of-source duties for the +opengrep binary itself. That is out of scope for M0.7 (the harness fetches the binary at setup time on +the operator's own machine and ships nothing), and it is already assigned: `plan/30-lane-b-detection.md` +B.2 owns `data/LICENSES/opengrep-binary-distribution.md`. Recorded here so the deferral is a decision +and not an oversight. + +--- + +## 2. Ruleset — `AikidoSec/opengrep-rules` @ `7ac79af` — **MIT** + +**Fetched:** +`https://raw.githubusercontent.com/AikidoSec/opengrep-rules/7ac79affecf709eb7263a243b518a417cd7e0ab2/LICENSE` +(1075 bytes, sha256 `3053445ee21294dbf1c714f45c0808aa3bb29ee60e0737efde63bcbb523ac8c8`, git blob +`b48a9af2d18b4847a0cfa4882d4aafa180052543`). The URL pins the **commit**, so this is the licence text +of the exact tree Anvil uses, not of whatever `main` becomes later. + +Verbatim from the LICENSE body: + +``` +MIT License + +Copyright (c) 2025 Aikido Security BV + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +``` + +**Finding: MIT.** No field-of-use restriction, no Commons Clause rider, no non-commercial clause. The +grant explicitly includes `distribute` and `sublicense`, so vendoring the rules into an Apache-2.0 +project is clean, subject only to MIT's attribution condition (retain the copyright notice and the +permission notice — `acquire.py` copies `LICENSE` into every checkout for exactly that reason). + +Corroborating metadata (`/repos/AikidoSec/opengrep-rules`, fetched in-session, **not** the finding): +`license.spdx_id = "MIT"`, `archived = false`, `default_branch = "main"`, +`pushed_at = 2026-06-04T16:08:29Z`, 37 stars. This matches `research/10` [S13] and `research/14` M6. + +**Pinned commit:** `7ac79affecf709eb7263a243b518a417cd7e0ab2` (2026-06-04T16:06:22Z, "Merge pull +request #2 from AikidoSec/kapyteinaikido-patch-1"). This is the current `main` head; the repo has 7 +commits total, one branch, and zero tags, so there is no release to pin to and the commit SHA is the +only stable handle. + +Each rule file also carries its own in-band licence declaration, which is unusual and worth recording +— `rules/github_workflow_prompt_injection/github_workflow_prompt_injection.yaml` contains: + +```yaml + license: MIT License (https://github.com/AikidoSec/opengrep-rules/blob/main/LICENSE) +``` + +--- + +## 3. Excluded — recorded so nobody re-adds them + +### `opengrep/opengrep-rules` — **HARD EXCLUDED**, `plan/00-SPINE.md` S5 + +Re-verified in-session 2026-08-06 via `/repos/opengrep/opengrep-rules`, independently reproducing all +four values from `research/10` [S12] and `research/14` (V6): + +| Field | Observed 2026-08-06 | +|---|---| +| `archived` | `true` | +| `license.spdx_id` | `NOASSERTION` | +| `stargazers_count` | 6 | +| `pushed_at` | 2025-11-28T13:17:29Z | + +Its LICENSE is LGPL-2.1 **plus a Commons Clause rider** removing the right to Sell the Software. +Commons Clause is not OSI-approved; redistributing these rules inside an OSI-licensed project is a +licence conflict. It is not used, not fetched, and not referenced by any code path here. The only +reason it appears in this repository at all is as a named exclusion in `MANIFEST.toml [[excluded.repos]]`, +so that a future contributor reaching for "the obvious substitute" hits a wall with a reason on it. + +### Semgrep-maintained rules — **HARD EXCLUDED**, `plan/00-SPINE.md` S5 + +Semgrep Rules License v.1.0 permits use "only for your own internal business purposes" and forbids +distribution. Out in every form, including rules derived by reading them. + +--- + +## 4. Coverage finding — measured, and it is the thing to actually worry about + +The licence question on `AikidoSec/opengrep-rules` is settled and clean. The **coverage** question is +not, and M0.7 measured it rather than assuming it. At the pinned commit the entire repository is: + +``` +LICENSE +README.md +rules/github_workflow_prompt_injection/github_workflow_prompt_injection.yaml +rules/npm_staged_publishing_missing/npm_staged_publishing_missing.yaml +``` + +Two rules. Both `languages: [yaml]`. Both `paths.include`-filtered to `.github/workflows/**` and +`.github/actions/**`. Neither performs taint analysis. Neither looks at application source code in any +language Anvil targets. + +`research/10` already hedged this — "Small and low-visibility, but legally unambiguous... rule coverage +unassessed" [S13] — and `plan/30-lane-b-detection.md` open issue 6 states outright that first-party +coverage "is unverified anywhere in" the corpus. This is the verification. The result is that the +recall tier's *rule corpus*, as picked, is a two-rule GitHub-Actions linter. + +This does **not** contradict `plan/00-SPINE.md` S4 or S5 on licence grounds — the picks are correct and +the exclusions hold. It bears on what INSTR-01 (candidates-per-scan, `plan/10-milestone0-evaluation.md` +M0.11) will actually measure: against this corpus, a repo with no GitHub Actions workflows yields +exactly zero candidates, and the adjudicator-precision case that `research/14` M6 says rests on this +tier existing has almost nothing to adjudicate. The engine is not the problem — opengrep's taint +support is real. The MIT rule corpus is the problem. + +Escalated to the orchestrator, not resolved here: M0.7's scope is acquisition, and choosing a different +or additional rule source is an S4/S5 component decision that this packet may not make. diff --git a/eval/tools/opengrep/MANIFEST.toml b/eval/tools/opengrep/MANIFEST.toml new file mode 100644 index 0000000..46470f9 --- /dev/null +++ b/eval/tools/opengrep/MANIFEST.toml @@ -0,0 +1,186 @@ +# Anvil M0.7 — pinned acquisition manifest for the deterministic recall tier. +# +# Scope: this file is DATA ONLY. It records exactly which artefacts Anvil's +# evaluation harness is allowed to fetch, and the checksum each fetch must match. +# Nothing here downloads anything; see anvil_opengrep/acquire.py. +# +# Provenance of every value below: fetched in-session 2026-08-06 from the GitHub +# REST API / raw.githubusercontent.com. See LICENSES.md for the exact endpoints +# and for the licence findings. Values recalled from memory are not permitted here. +# +# Spine references: +# plan/00-SPINE.md S4 — source-recall pick: opengrep engine + AikidoSec/opengrep-rules +# plan/00-SPINE.md S5 — hard exclusions (see [excluded] at the bottom of this file) +# plan/00-SPINE.md S7 — supply-chain pinning: pin by commit SHA, diff before promotion +# plan/00-SPINE.md S12 — opengrep has zero bindings in any language; subprocess only + +schema_version = 1 +generated_utc = "2026-08-06" +generated_by = "Anvil implementation plan, step M0.7" + +# --------------------------------------------------------------------------- +# Engine: opengrep CLI (LGPL-2.1). Invoked as a SUBPROCESS ONLY, never linked. +# --------------------------------------------------------------------------- +[engine] +name = "opengrep" +repo = "https://github.com/opengrep/opengrep" +version = "v1.26.0" +release_tag = "v1.26.0" +release_commit_sha = "1bef4ea4ff3264754132eec823b5b1d8cde3e4ee" +release_published_utc = "2026-07-24T20:00:53Z" +release_html_url = "https://github.com/opengrep/opengrep/releases/tag/v1.26.0" +license_spdx = "LGPL-2.1" +license_url = "https://raw.githubusercontent.com/opengrep/opengrep/v1.26.0/LICENSE" +license_verified_utc = "2026-08-06" +# Linkage posture, restated here because it is a licence-compliance invariant: +linkage = "subprocess" +linkage_note = """opengrep is an OCaml CLI with zero bindings in any language (plan/00-SPINE.md S12). +Anvil execs the binary and reads its stdout. No FFI, no cgo, no shared-object load, no static link. +The LGPL-2.1 "combined work" analysis therefore never engages for the running system.""" + +# Asset checksums. Source: GitHub Releases API `assets[].digest`, which GitHub +# computes over the stored asset bytes. Fetched in-session. +# +# HONESTY NOTE, do not delete: these digests were read from the GitHub API. As of +# this packet no asset has been downloaded and independently re-hashed on this +# host (the orchestrator explicitly forbade downloading the binary in M0.7). +# acquire.py enforces the digest at fetch time, which is the point at which the +# claim first becomes load-bearing. +[[engine.assets]] +platform = "linux-x86_64-glibc" +filename = "opengrep_manylinux_x86" +url = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_manylinux_x86" +sha256 = "40c21299eeddabf743b856daa843d24f9d4a027130671cd45b3b21776fd9ab26" +size_bytes = 41867160 +sigstore_sig = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_manylinux_x86.sig" +sigstore_cert = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_manylinux_x86.cert" + +[[engine.assets]] +platform = "linux-aarch64-glibc" +filename = "opengrep_manylinux_aarch64" +url = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_manylinux_aarch64" +sha256 = "3042a3b1aa98fa93407b9d66a45ab1f179b5b367e76965f56afdbd2c038fb1fa" +size_bytes = 43569499 +sigstore_sig = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_manylinux_aarch64.sig" +sigstore_cert = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_manylinux_aarch64.cert" + +[[engine.assets]] +platform = "linux-x86_64-musl" +filename = "opengrep_musllinux_x86" +url = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_musllinux_x86" +sha256 = "18aeca114221e2816ec26e1a731f1a2583408c8e4578cd868cd2d47c12fd29f8" +size_bytes = 44153516 +sigstore_sig = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_musllinux_x86.sig" +sigstore_cert = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_musllinux_x86.cert" + +[[engine.assets]] +platform = "linux-aarch64-musl" +filename = "opengrep_musllinux_aarch64" +url = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_musllinux_aarch64" +sha256 = "d4e20ac57b6f9bb32c2b0ffc0501b8c6acb92ecee60f11f1cd72db9b11647857" +size_bytes = 45794214 +sigstore_sig = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_musllinux_aarch64.sig" +sigstore_cert = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_musllinux_aarch64.cert" + +[[engine.assets]] +platform = "darwin-arm64" +filename = "opengrep_osx_arm64" +url = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_osx_arm64" +sha256 = "513ff8491f7254c9a672cf8421136a537eb53b2a8af748568bd697acdc59eefe" +size_bytes = 43665440 +sigstore_sig = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_osx_arm64.sig" +sigstore_cert = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_osx_arm64.cert" + +[[engine.assets]] +platform = "darwin-x86_64" +filename = "opengrep_osx_x86" +url = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_osx_x86" +sha256 = "36c00a2b6eeb45796275e69cb8f74ef27c42724a1b3c98f6c8d861bad7a8529d" +size_bytes = 44252144 +sigstore_sig = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_osx_x86.sig" +sigstore_cert = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_osx_x86.cert" + +[[engine.assets]] +platform = "windows-x86_64" +filename = "opengrep_windows_x86.exe" +url = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_windows_x86.exe" +sha256 = "4e6c0e201982cd72ca4aff5798a2ff133e17de8af3b00b460238fdda4dd266e3" +size_bytes = 49845248 +sigstore_sig = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_windows_x86.exe.sig" +sigstore_cert = "https://github.com/opengrep/opengrep/releases/download/v1.26.0/opengrep_windows_x86.exe.cert" + +# --------------------------------------------------------------------------- +# Ruleset: AikidoSec/opengrep-rules (MIT). The ONLY permitted rule source. +# --------------------------------------------------------------------------- +[ruleset] +name = "AikidoSec/opengrep-rules" +repo = "https://github.com/AikidoSec/opengrep-rules" +# S7 supply-chain pinning: commit SHA, never a branch name, never `main`. +commit_sha = "7ac79affecf709eb7263a243b518a417cd7e0ab2" +commit_utc = "2026-06-04T16:06:22Z" +branch_observed = "main" +archived = false +license_spdx = "MIT" +license_url = "https://raw.githubusercontent.com/AikidoSec/opengrep-rules/7ac79affecf709eb7263a243b518a417cd7e0ab2/LICENSE" +license_verified_utc = "2026-08-06" +license_holder = "Aikido Security BV" +# Advisory only. GitHub codeload tarballs are re-gzipped and are NOT byte-stable +# across time; the authoritative pin is commit_sha + the per-file blob SHAs below. +tarball_url = "https://codeload.github.com/AikidoSec/opengrep-rules/tar.gz/7ac79affecf709eb7263a243b518a417cd7e0ab2" +tarball_sha256_observed = "8b683f8e5907b368763c8e05bdbf298607da22ad4e34625166ce950bfe8d1e51" +tarball_sha256_authoritative = false + +# Git blob SHAs (sha1 of "blob \0" + content). Content-addressed and stable +# forever, so a checkout can be verified file-by-file without trusting git itself. +[[ruleset.files]] +path = "LICENSE" +blob_sha1 = "b48a9af2d18b4847a0cfa4882d4aafa180052543" +size_bytes = 1075 + +[[ruleset.files]] +path = "README.md" +blob_sha1 = "16937a1ad926b171423702f8604964d50063f21f" +size_bytes = 295 + +[[ruleset.files]] +path = "rules/github_workflow_prompt_injection/github_workflow_prompt_injection.yaml" +blob_sha1 = "9e133e0d61fba2fc86ce1c97197355bd14b57c71" +size_bytes = 1539 + +[[ruleset.files]] +path = "rules/npm_staged_publishing_missing/npm_staged_publishing_missing.yaml" +blob_sha1 = "bcfbe4803a0d0f3142afcd3dccd96256e9318adc" +size_bytes = 647 + +# Corpus size, measured — not estimated. This is load-bearing for INSTR-01. +[ruleset.coverage] +rule_files = 2 +rule_ids = ["github_workflow_prompt_injection", "npm_staged_publishing_missing"] +languages = ["yaml"] +path_filters = [".github/workflows/**", ".github/actions/**"] +total_repo_commits = 7 +tags = 0 +warning = """MEASURED, NOT ESTIMATED: at the pinned commit this ruleset contains exactly TWO rules, +both YAML-only and both path-filtered to .github/workflows and .github/actions. It performs no +taint analysis and covers no general-purpose source language. See LICENSES.md section 4 and the +M0.7 report: this is a coverage finding the orchestrator must resolve before INSTR-01 is read as +a recall measurement of anything.""" + +# --------------------------------------------------------------------------- +# S5 hard exclusions. Recorded so a future contributor cannot "helpfully" add them. +# --------------------------------------------------------------------------- +[excluded] +[[excluded.repos]] +repo = "opengrep/opengrep-rules" +reason = "plan/00-SPINE.md S5 hard exclusion" +observed_utc = "2026-08-06" +archived = true +spdx_id = "NOASSERTION" +stars = 6 +last_push = "2025-11-28T13:17:29Z" +detail = "LGPL-2.1 plus a Commons Clause rider; Commons Clause is not OSI-approved and redistribution inside an OSI-licensed project is a licence conflict. Never vendor, never --config against it." + +[[excluded.repos]] +repo = "semgrep/semgrep-rules" +reason = "plan/00-SPINE.md S5 hard exclusion" +detail = "Semgrep Rules License v.1.0 — internal business use only, no redistribution, no service. Any Semgrep-maintained ruleset is out, in every form." diff --git a/eval/tools/opengrep/README.md b/eval/tools/opengrep/README.md new file mode 100644 index 0000000..d465bdf --- /dev/null +++ b/eval/tools/opengrep/README.md @@ -0,0 +1,113 @@ +# `eval/tools/opengrep` — the deterministic recall tier, pinned + +Anvil step **M0.7** (`plan/10-milestone0-evaluation.md`). Acquires and invokes the recall-tier +stand-in that INSTR-01 (candidates-per-scan, `plan/00-SPINE.md` S2) measures against: + +| | Pick | Licence | Pin | +|---|---|---|---| +| Engine | `opengrep/opengrep` | LGPL-2.1 | release tag `v1.26.0`, per-asset sha256 | +| Rules | `AikidoSec/opengrep-rules` | MIT | commit `7ac79affecf709eb7263a243b518a417cd7e0ab2` | + +Licence findings, quoted from the LICENSE file bodies, are in [`LICENSES.md`](LICENSES.md). The pins +themselves are in [`MANIFEST.toml`](MANIFEST.toml). **Read `LICENSES.md` section 4 before reading any +INSTR-01 number** — the MIT rule corpus is two rules, and that changes what a candidate count means. + +## Use + +```bash +cd eval/tools/opengrep + +python -m anvil_opengrep.acquire --rules # clone + verify the pinned ruleset (a few KB) +python -m anvil_opengrep.acquire --engine # download + sha256-verify the engine (~42 MB) +python -m anvil_opengrep.acquire --verify-only # check what is on disk, download nothing + +python smoke.py # end-to-end run against the sample repo +python -m pytest tests -q # unit tests; smoke tests skip if artefacts absent +``` + +```python +from anvil_opengrep import OpengrepRunner + +result = OpengrepRunner().scan("/path/to/repo") +print(result.candidate_count) # the INSTR-01 quantity +for finding in result.findings: + print(finding.rule_id, finding.candidate_key) +``` + +`smoke.py` exit codes: `0` expected candidates, `1` pin drift, `2` engine/ruleset absent, `3` scan failed. + +## Subprocess only, never linked + +opengrep is an OCaml CLI with **zero bindings in any language** (`plan/00-SPINE.md` S12), so +`subprocess.run` is not a stylistic choice — it is the only mechanism that exists. `runner.py` contains +no FFI, no `ctypes`, no dynamic load. `manifest.py` refuses to load a manifest whose +`engine.linkage` is anything other than `"subprocess"`, so the invariant fails at load time rather +than in review. This is the same line `plan/30-lane-b-detection.md` B.1 holds on the Go side. + +The LGPL obligation that *does* attach — shipping the compiled binary inside a distributed container +image is conveyance — is out of scope here and assigned to B.2. See `LICENSES.md` section 1. + +## Failing loudly + +Every failure in this package raises. There is no path that returns an empty finding list because +something was missing: + +| Condition | Behaviour | +|---|---| +| engine binary absent / not executable | `EngineNotAvailable` | +| ruleset checkout absent | `RulesetNotAvailable` | +| asset sha256 or rule blob SHA mismatch | `ChecksumMismatch`, never retried | +| exit code outside `{0, 1}` | `OpengrepRunError` with stderr attached | +| no SARIF written, or unparseable | `OpengrepOutputError` | +| `--config` naming an S5-excluded repo | `ForbiddenRuleSource` | + +The reason is narrow and specific. INSTR-01 reads candidate counts as a measurement. If a missing +binary could produce "0 candidates", a broken install and a genuinely clean repo would be recorded +identically, and the experiment would be quietly worthless. So `ScanResult.findings == ()` means +exactly one thing: the pinned engine ran the pinned rules over the target and matched nothing. + +## S5 hard exclusions, enforced in code + +`plan/00-SPINE.md` S5 excludes `opengrep/opengrep-rules` (archived, `NOASSERTION`, LGPL-2.1 + +Commons Clause) and every Semgrep-maintained ruleset (internal business use only). +`manifest.assert_rule_source_permitted()` runs against the manifest at load time **and** against +whatever `ruleset_path` a caller actually hands `OpengrepRunner`, because an exclusion enforced only +on the happy path is not enforced. `tests/test_manifest.py` covers both repos and both path +separators. + +One consequence worth knowing: the vendored ruleset directory is deliberately named +`vendor/aikido-opengrep-rules/`, not `vendor/opengrep/opengrep-rules/`, since the latter contains the +excluded slug as a substring and the guard would refuse it. + +## The fixture is materialized, not committed + +Both pinned rules carry `paths.include` filters on `.github/workflows/**`, so a fixture that exercises +them must live at that path. Rather than create a real `.github/workflows/` directory inside the Anvil +repository — where actionlint, Dependabot, and anything globbing `**/.github/workflows` would find +inert fixture files — the fixtures are stored flat in `fixtures/` and assembled into the required +layout in a temp directory at scan time by `anvil_opengrep.fixtures.materialize_sample_repo()`. + +That also avoids a subtler trap: opengrep scans **only git-tracked files** when the target is a git +repository. A fixture sitting untracked inside this repo would be silently skipped. A freshly +materialized plain directory is not a git repo, so everything in it is scanned. + +The sample repo carries two positive cases (one per rule), one clean workflow, and `src/decoy_app.py` +— an unguarded command injection that the pinned ruleset is *expected not to find*. That non-result +is the coverage finding from `LICENSES.md` section 4 in executable form. + +## Layout + +``` +MANIFEST.toml pins: release tag, per-asset sha256, ruleset commit + blob SHAs, S5 exclusions +LICENSES.md licence findings quoted from LICENSE bodies; the coverage finding +smoke.py end-to-end driver +anvil_opengrep/ + manifest.py parse + validate the pins; the S5 guard + acquire.py pinned fetch + checksum verification (nothing runs on import) + runner.py subprocess wrapper; SARIF 2.1.0 parsing + fixtures.py materializes the sample repo + errors.py every failure mode +fixtures/ flat fixture sources +tests/ unit tests + the smoke test (skips loudly without artefacts) +vendor/ acquired artefacts — gitignored, never committed +``` diff --git a/eval/tools/opengrep/anvil_opengrep/__init__.py b/eval/tools/opengrep/anvil_opengrep/__init__.py new file mode 100644 index 0000000..4876735 --- /dev/null +++ b/eval/tools/opengrep/anvil_opengrep/__init__.py @@ -0,0 +1,42 @@ +"""Anvil M0.7 — pinned acquisition + subprocess wrapper for the opengrep engine. + +The deterministic recall tier (plan/00-SPINE.md S4) is the opengrep engine +(LGPL-2.1) driven by AikidoSec/opengrep-rules (MIT). This package is the +evaluation harness's side of that: it reads MANIFEST.toml, fetches exactly the +pinned artefacts and nothing else, and invokes the engine as a subprocess. + +opengrep is invoked as a SUBPROCESS ONLY, never linked. It is an OCaml CLI with +zero bindings in any language (plan/00-SPINE.md S12), so subprocess is not a +preference here, it is the only option that exists. +""" + +from __future__ import annotations + +from .errors import ( + ChecksumMismatch, + EngineNotAvailable, + ForbiddenRuleSource, + ManifestError, + OpengrepError, + OpengrepOutputError, + OpengrepRunError, + RulesetNotAvailable, +) +from .manifest import Manifest, load_manifest +from .runner import Finding, OpengrepRunner, ScanResult + +__all__ = [ + "ChecksumMismatch", + "EngineNotAvailable", + "Finding", + "ForbiddenRuleSource", + "Manifest", + "ManifestError", + "OpengrepError", + "OpengrepOutputError", + "OpengrepRunError", + "OpengrepRunner", + "RulesetNotAvailable", + "ScanResult", + "load_manifest", +] diff --git a/eval/tools/opengrep/anvil_opengrep/acquire.py b/eval/tools/opengrep/anvil_opengrep/acquire.py new file mode 100644 index 0000000..8c1ae6d --- /dev/null +++ b/eval/tools/opengrep/anvil_opengrep/acquire.py @@ -0,0 +1,307 @@ +"""Pinned acquisition of the opengrep engine and the AikidoSec ruleset. + +Nothing here runs on import, and nothing here is invoked by the runner. The +harness operator runs `python -m anvil_opengrep.acquire` once, deliberately. + +Two invariants: + +1. **Only what MANIFEST.toml pins.** No "latest", no version resolution, no + redirect to an unpinned URL. The URL comes out of the manifest verbatim. +2. **Checksum or nothing.** The engine asset must match its pinned sha256 and + the ruleset checkout must match its pinned commit SHA and per-file git blob + SHAs. A mismatch raises ChecksumMismatch and deletes the partial download; it + is never retried, never warned about, never ignored. + +The digests in MANIFEST.toml were read from the GitHub Releases API in-session +and have not yet been confirmed by downloading the asset (see the honesty note +in the manifest). This module is where that confirmation happens. +""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import shutil +import subprocess +import sys +import tempfile +import urllib.request +from pathlib import Path + +from .errors import ChecksumMismatch, OpengrepError, RulesetNotAvailable +from .manifest import Manifest, assert_rule_source_permitted, load_manifest + +# Where acquired artefacts land. Kept out of git (see .gitignore in this dir). +DEFAULT_VENDOR_DIR = Path(__file__).resolve().parent.parent / "vendor" +ENGINE_SUBDIR = "engine" +# NOT "opengrep-rules" and NOT nested under a dir called "opengrep": the S5 +# substring guard in manifest.assert_rule_source_permitted would (correctly) +# refuse a path containing "opengrep/opengrep-rules". +RULES_SUBDIR = "aikido-opengrep-rules" + +_CHUNK = 1 << 20 + + +def engine_path(vendor_dir: Path | None = None, manifest: Manifest | None = None) -> Path: + """Filesystem location the pinned engine binary is installed to.""" + manifest = manifest or load_manifest() + vendor = Path(vendor_dir) if vendor_dir else DEFAULT_VENDOR_DIR + return vendor / ENGINE_SUBDIR / manifest.engine_version / manifest.asset_for().filename + + +def ruleset_path(vendor_dir: Path | None = None, manifest: Manifest | None = None) -> Path: + """Filesystem location the pinned ruleset checkout is installed to.""" + manifest = manifest or load_manifest() + vendor = Path(vendor_dir) if vendor_dir else DEFAULT_VENDOR_DIR + return vendor / RULES_SUBDIR / manifest.ruleset_commit_sha + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(_CHUNK), b""): + digest.update(chunk) + return digest.hexdigest() + + +def git_blob_sha1(path: Path) -> str: + """Compute the git blob object id for a file: sha1(b"blob \\0" + content). + + Content-addressed and stable forever, unlike a codeload tarball, so a + checkout can be verified file-by-file without trusting git or the network. + """ + data = Path(path).read_bytes() + header = f"blob {len(data)}\0".encode() + return hashlib.sha1(header + data).hexdigest() # noqa: S324 - git's object id, not a security hash + + +def fetch_engine( + vendor_dir: Path | None = None, + manifest: Manifest | None = None, + platform_key: str | None = None, +) -> Path: + """Download the pinned opengrep binary for this host and verify its sha256.""" + manifest = manifest or load_manifest() + asset = manifest.asset_for(platform_key) + vendor = Path(vendor_dir) if vendor_dir else DEFAULT_VENDOR_DIR + dest_dir = vendor / ENGINE_SUBDIR / manifest.engine_version + dest = dest_dir / asset.filename + + if dest.is_file(): + actual = sha256_file(dest) + if actual == asset.sha256: + return dest + raise ChecksumMismatch( + f"existing {dest} has sha256 {actual}, manifest pins {asset.sha256}. " + "Delete it deliberately; this file is not overwritten automatically." + ) + + dest_dir.mkdir(parents=True, exist_ok=True) + tmp_fd, tmp_name = tempfile.mkstemp(dir=str(dest_dir), prefix=".partial-") + os.close(tmp_fd) + tmp = Path(tmp_name) + try: + # The URL is manifest-pinned; it is never derived from user input. + with urllib.request.urlopen(asset.url) as response, tmp.open("wb") as out: # noqa: S310 + shutil.copyfileobj(response, out, _CHUNK) + actual = sha256_file(tmp) + if actual != asset.sha256: + raise ChecksumMismatch( + f"{asset.url}\n expected sha256 {asset.sha256}\n actual sha256 {actual}\n" + "Supply-chain mismatch. Not retried. Investigate before proceeding." + ) + size = tmp.stat().st_size + if size != asset.size_bytes: + raise ChecksumMismatch( + f"{asset.url}: size {size} != pinned {asset.size_bytes} " + "(digest matched, which is odd)" + ) + tmp.replace(dest) + finally: + if tmp.exists(): + tmp.unlink() + + if os.name != "nt": + dest.chmod(dest.stat().st_mode | 0o111) + return dest + + +def fetch_ruleset(vendor_dir: Path | None = None, manifest: Manifest | None = None) -> Path: + """Clone AikidoSec/opengrep-rules and hard-check out the pinned commit SHA. + + git is used only as a transport. Trust comes from `verify_ruleset`, which + re-derives every file's git blob id locally. + """ + manifest = manifest or load_manifest() + assert_rule_source_permitted(manifest.ruleset_repo) + dest = ruleset_path(vendor_dir, manifest) + if dest.is_dir(): + verify_ruleset(dest, manifest) + return dest + + dest.parent.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(dir=str(dest.parent), prefix=".clone-")) + try: + # core.autocrlf=false / core.eol=lf are load-bearing, not cosmetic. With + # git's Windows defaults the working tree gets CRLF line endings and every + # file's bytes stop matching its blob id, so verify_ruleset correctly but + # uselessly rejects an otherwise-pristine checkout. Forcing LF makes the + # checkout byte-identical to the pinned objects on every platform. + _git( + [ + "-c", + "core.autocrlf=false", + "-c", + "core.eol=lf", + "clone", + "--quiet", + "--no-checkout", + manifest.ruleset_repo, + str(staging / "repo"), + ] + ) + repo = staging / "repo" + _git( + [ + "-c", + "core.autocrlf=false", + "-c", + "core.eol=lf", + "-C", + str(repo), + "checkout", + "--quiet", + "--detach", + manifest.ruleset_commit_sha, + ] + ) + head = _git(["-C", str(repo), "rev-parse", "HEAD"]).strip() + if head != manifest.ruleset_commit_sha: + raise ChecksumMismatch( + f"checked out HEAD {head} != pinned {manifest.ruleset_commit_sha}" + ) + _rmtree(repo / ".git") + repo.replace(dest) + finally: + _rmtree(staging) + + verify_ruleset(dest, manifest) + return dest + + +def verify_ruleset(path: Path, manifest: Manifest | None = None) -> None: + """Re-derive each pinned file's git blob id from local bytes. + + Raises RulesetNotAvailable if a pinned file is missing, ChecksumMismatch if + its content differs from the pin. The MIT LICENSE file is included in the + pin set on purpose: MIT's only condition is that the notice travels with the + rules, so an absent LICENSE is a compliance failure, not a cosmetic one. + """ + manifest = manifest or load_manifest() + root = Path(path) + if not root.is_dir(): + raise RulesetNotAvailable( + f"pinned ruleset checkout missing at {root}. " + "Run: python -m anvil_opengrep.acquire --rules" + ) + for entry in manifest.ruleset_files: + target = root / entry.path + if not target.is_file(): + raise RulesetNotAvailable(f"pinned ruleset file missing: {target}") + actual = git_blob_sha1(target) + if actual != entry.blob_sha1: + raise ChecksumMismatch( + f"{target}\n expected git blob {entry.blob_sha1}\n actual git blob {actual}\n" + f"The pinned ruleset at {manifest.ruleset_commit_sha} has been modified locally." + ) + + +def _rmtree(path: Path) -> None: + """rmtree that survives git's read-only pack files on Windows.""" + + def _on_error(func, target, _exc): # pragma: no cover - platform dependent + try: + os.chmod(target, 0o700) + func(target) + except OSError: + pass + + if sys.version_info >= (3, 12): + shutil.rmtree(path, onexc=lambda f, t, e: _on_error(f, t, e)) + else: # pragma: no cover - Python 3.11 fallback + shutil.rmtree(path, onerror=lambda f, t, e: _on_error(f, t, e)) + + +def _git(args: list[str]) -> str: + if shutil.which("git") is None: + raise OpengrepError("git is not on PATH; it is required to fetch the pinned ruleset") + proc = subprocess.run( # noqa: S603 - fixed argv, no shell + ["git", *args], capture_output=True, text=True, check=False + ) + if proc.returncode != 0: + raise OpengrepError( + f"git {' '.join(args)} failed ({proc.returncode}): {proc.stderr.strip()}" + ) + return proc.stdout + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m anvil_opengrep.acquire", + description="Fetch the MANIFEST.toml-pinned opengrep engine and AikidoSec ruleset.", + ) + parser.add_argument("--engine", action="store_true", help="fetch the pinned engine binary") + parser.add_argument("--rules", action="store_true", help="fetch the pinned ruleset checkout") + parser.add_argument("--vendor-dir", default=None, help="override the vendor directory") + parser.add_argument( + "--verify-only", + action="store_true", + help="verify what is already on disk; download nothing", + ) + args = parser.parse_args(argv) + + if not args.engine and not args.rules: + args.engine = args.rules = True + + manifest = load_manifest() + vendor = Path(args.vendor_dir) if args.vendor_dir else None + + print(f"manifest : {manifest.path}") + print( + f"engine : {manifest.engine_repo} {manifest.engine_version} " + f"({manifest.engine_license})" + ) + print( + f"ruleset : {manifest.ruleset_name} @ {manifest.ruleset_commit_sha} " + f"({manifest.ruleset_license})" + ) + + try: + if args.rules: + target = ruleset_path(vendor, manifest) + if args.verify_only: + verify_ruleset(target, manifest) + print(f"ruleset verified: {target}") + else: + print(f"ruleset at : {fetch_ruleset(vendor, manifest)}") + if args.engine: + target = engine_path(vendor, manifest) + if args.verify_only: + if not target.is_file(): + raise OpengrepError(f"engine binary absent at {target}") + actual = sha256_file(target) + expected = manifest.asset_for().sha256 + if actual != expected: + raise ChecksumMismatch(f"{target}: {actual} != pinned {expected}") + print(f"engine verified : {target}") + else: + print(f"engine at : {fetch_engine(vendor, manifest)}") + except OpengrepError as exc: + print(f"FAILED: {type(exc).__name__}: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/tools/opengrep/anvil_opengrep/errors.py b/eval/tools/opengrep/anvil_opengrep/errors.py new file mode 100644 index 0000000..45d2965 --- /dev/null +++ b/eval/tools/opengrep/anvil_opengrep/errors.py @@ -0,0 +1,52 @@ +"""Failure modes for the opengrep acquisition + invocation path. + +Every one of these is a hard stop. Nothing in this package degrades gracefully: +a missing binary, a missing ruleset, a checksum mismatch, or an unparseable +result must be loud, because the alternative is INSTR-01 quietly reporting +"0 candidates" for a reason that has nothing to do with the target repo. +""" + +from __future__ import annotations + + +class OpengrepError(Exception): + """Base class for every failure in this package.""" + + +class ManifestError(OpengrepError): + """MANIFEST.toml is missing, malformed, or missing a required pin.""" + + +class EngineNotAvailable(OpengrepError): + """The pinned opengrep binary is not present or not executable. + + Raised instead of returning an empty result set. See README.md ("Failing + loudly") for why this is never downgraded to a warning. + """ + + +class RulesetNotAvailable(OpengrepError): + """The pinned AikidoSec/opengrep-rules checkout is absent or unverified.""" + + +class ChecksumMismatch(OpengrepError): + """A fetched artefact did not match its pinned digest. + + This is a supply-chain event, not a retry-able network error. + """ + + +class OpengrepRunError(OpengrepError): + """opengrep ran but failed: non-recoverable exit code, or timeout.""" + + +class OpengrepOutputError(OpengrepError): + """opengrep exited plausibly but its stdout was not parseable JSON.""" + + +class ForbiddenRuleSource(OpengrepError): + """An S5 hard-excluded rule source was passed to the runner. + + plan/00-SPINE.md S5: never opengrep/opengrep-rules (archived, NOASSERTION, + LGPL-2.1 + Commons Clause), never any Semgrep-maintained ruleset. + """ diff --git a/eval/tools/opengrep/anvil_opengrep/fixtures.py b/eval/tools/opengrep/anvil_opengrep/fixtures.py new file mode 100644 index 0000000..b3cb623 --- /dev/null +++ b/eval/tools/opengrep/anvil_opengrep/fixtures.py @@ -0,0 +1,61 @@ +"""Materialize the smoke-test sample repo into a throwaway directory. + +Why materialize instead of committing the tree as-is: both pinned AikidoSec +rules carry `paths.include` filters on `.github/workflows/**`, so a fixture that +exercises them has to live at that path. Creating a real `.github/workflows/` +directory anywhere inside the Anvil repository would put inert fixture files in +front of every tool that globs `**/.github/workflows` — actionlint, Dependabot, +workflow scanners. So the fixture files are stored flat under `fixtures/` and +assembled into the required layout in a temp directory at scan time. + +Second benefit: the materialized tree is not a git repository, and opengrep +defaults to scanning only git-tracked files when the target is one. A plain +directory sidesteps that entirely. +""" + +from __future__ import annotations + +import shutil +import tempfile +from pathlib import Path + +FIXTURE_SOURCE_DIR = Path(__file__).resolve().parent.parent / "fixtures" + +# source filename in fixtures/ -> path inside the materialized sample repo +SAMPLE_REPO_LAYOUT: dict[str, str] = { + "workflow_prompt_injection.yml": ".github/workflows/workflow_prompt_injection.yml", + "workflow_npm_publish.yml": ".github/workflows/workflow_npm_publish.yml", + "clean_workflow.yml": ".github/workflows/clean_workflow.yml", + "decoy_app.py": "src/decoy_app.py", +} + +# What the pinned ruleset (AikidoSec/opengrep-rules @ 7ac79af) is expected to +# find. Asserted by the smoke test so that a silently-changed pin is caught. +EXPECTED_RULE_IDS: frozenset[str] = frozenset( + {"github_workflow_prompt_injection", "npm_staged_publishing_missing"} +) + +# Files the ruleset must NOT flag. A hit here means misattribution or pin drift. +EXPECTED_CLEAN_PATHS: frozenset[str] = frozenset( + {".github/workflows/clean_workflow.yml", "src/decoy_app.py"} +) + + +def materialize_sample_repo(dest: str | Path | None = None) -> Path: + """Write the sample repo into `dest` (or a fresh temp dir) and return its root. + + The caller owns cleanup when it passes `dest`; when it does not, the + directory is a `tempfile.mkdtemp` the caller should remove. + """ + root = ( + Path(dest) if dest is not None else Path(tempfile.mkdtemp(prefix="anvil-opengrep-fixture-")) + ) + root.mkdir(parents=True, exist_ok=True) + for source_name, relative in SAMPLE_REPO_LAYOUT.items(): + source = FIXTURE_SOURCE_DIR / source_name + if not source.is_file(): + raise FileNotFoundError(f"fixture source missing: {source}") + target = root / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + return root diff --git a/eval/tools/opengrep/anvil_opengrep/manifest.py b/eval/tools/opengrep/anvil_opengrep/manifest.py new file mode 100644 index 0000000..fd3833a --- /dev/null +++ b/eval/tools/opengrep/anvil_opengrep/manifest.py @@ -0,0 +1,237 @@ +"""Load and validate MANIFEST.toml — the single source of truth for what may be fetched. + +Stdlib only (`tomllib`, Python 3.11+). The evaluation harness must be able to +read its own pins without any third-party dependency being installed first. +""" + +from __future__ import annotations + +import platform +import sys +import tomllib +from dataclasses import dataclass +from pathlib import Path + +from .errors import ManifestError + +# Repository roots that plan/00-SPINE.md S5 hard-excludes. Matched as substrings +# against any rule-source path or URL the caller supplies. Cheap, and it turns a +# licence violation into an exception instead of a code review someone skipped. +FORBIDDEN_RULE_SOURCES: tuple[str, ...] = ( + "opengrep/opengrep-rules", + "opengrep-rules-archived", + "semgrep/semgrep-rules", + "semgrep-rules", +) + +# The one permitted rule source, by repo slug. +PERMITTED_RULE_SOURCE = "AikidoSec/opengrep-rules" + +DEFAULT_MANIFEST_PATH = Path(__file__).resolve().parent.parent / "MANIFEST.toml" + + +@dataclass(frozen=True) +class EngineAsset: + platform: str + filename: str + url: str + sha256: str + size_bytes: int + sigstore_sig: str | None = None + sigstore_cert: str | None = None + + +@dataclass(frozen=True) +class RulesetFile: + path: str + blob_sha1: str + size_bytes: int + + +@dataclass(frozen=True) +class Manifest: + path: Path + engine_version: str + engine_repo: str + engine_commit_sha: str + engine_license: str + engine_assets: tuple[EngineAsset, ...] + ruleset_name: str + ruleset_repo: str + ruleset_commit_sha: str + ruleset_license: str + ruleset_files: tuple[RulesetFile, ...] + ruleset_rule_ids: tuple[str, ...] + + def asset_for(self, platform_key: str | None = None) -> EngineAsset: + """Return the pinned asset for a platform key, defaulting to this host.""" + key = platform_key or current_platform_key() + for asset in self.engine_assets: + if asset.platform == key: + return asset + known = ", ".join(sorted(a.platform for a in self.engine_assets)) + raise ManifestError( + f"no pinned opengrep asset for platform {key!r}. Pinned platforms: {known}" + ) + + +def current_platform_key() -> str: + """Map this interpreter's host to a MANIFEST.toml `engine.assets.platform` key.""" + machine = platform.machine().lower() + if machine in {"amd64", "x86_64"}: + arch = "x86_64" + elif machine in {"arm64", "aarch64"}: + arch = "aarch64" if sys.platform.startswith("linux") else "arm64" + else: + raise ManifestError(f"unsupported CPU architecture for opengrep: {machine!r}") + + if sys.platform.startswith("linux"): + # libc flavour matters: the manylinux build will not run on musl. + libc = "musl" if _is_musl() else "glibc" + return f"linux-{arch}-{libc}" + if sys.platform == "darwin": + return f"darwin-{'arm64' if arch == 'arm64' else 'x86_64'}" + if sys.platform in {"win32", "cygwin"}: + return "windows-x86_64" + raise ManifestError(f"unsupported platform for opengrep: {sys.platform!r}") + + +def _is_musl() -> bool: + try: + libc, _ = platform.libc_ver() + except (OSError, ValueError): # pragma: no cover - platform dependent + return False + if libc: + return "musl" in libc.lower() + # platform.libc_ver() returns ("", "") on musl systems; the marker file is + # the pragmatic fallback. + return any(Path("/lib").glob("ld-musl-*.so.1")) + + +def _require(table: dict, key: str, where: str): + if key not in table: + raise ManifestError(f"MANIFEST.toml: missing required key {where}.{key}") + return table[key] + + +def load_manifest(path: str | Path | None = None) -> Manifest: + """Parse and validate MANIFEST.toml. + + Validation is deliberately strict: an under-specified pin is a supply-chain + hole, and a manifest that parses but omits a SHA is worse than one that + fails to parse, because it looks fine. + """ + manifest_path = Path(path) if path is not None else DEFAULT_MANIFEST_PATH + if not manifest_path.is_file(): + raise ManifestError(f"MANIFEST.toml not found at {manifest_path}") + + try: + with manifest_path.open("rb") as handle: + data = tomllib.load(handle) + except tomllib.TOMLDecodeError as exc: + raise ManifestError(f"MANIFEST.toml is not valid TOML: {exc}") from exc + + engine = _require(data, "engine", "") + ruleset = _require(data, "ruleset", "") + + linkage = engine.get("linkage") + if linkage != "subprocess": + raise ManifestError( + "MANIFEST.toml: engine.linkage must be 'subprocess'. " + "plan/00-SPINE.md S12: opengrep has zero bindings in any language; " + "linking it is not merely discouraged, it is impossible, and claiming " + "otherwise in the manifest means the manifest is wrong." + ) + + raw_assets = engine.get("assets") or [] + if not raw_assets: + raise ManifestError("MANIFEST.toml: engine.assets is empty; nothing is pinned") + assets = [] + for index, raw in enumerate(raw_assets): + where = f"engine.assets[{index}]" + sha256 = str(_require(raw, "sha256", where)) + if len(sha256) != 64 or any(c not in "0123456789abcdef" for c in sha256): + raise ManifestError(f"{where}.sha256 is not a lowercase hex sha256: {sha256!r}") + assets.append( + EngineAsset( + platform=str(_require(raw, "platform", where)), + filename=str(_require(raw, "filename", where)), + url=str(_require(raw, "url", where)), + sha256=sha256, + size_bytes=int(_require(raw, "size_bytes", where)), + sigstore_sig=raw.get("sigstore_sig"), + sigstore_cert=raw.get("sigstore_cert"), + ) + ) + + ruleset_repo = str(_require(ruleset, "repo", "ruleset")) + ruleset_name = str(_require(ruleset, "name", "ruleset")) + assert_rule_source_permitted(ruleset_repo) + assert_rule_source_permitted(ruleset_name) + if PERMITTED_RULE_SOURCE.lower() not in ruleset_name.lower(): + raise ManifestError( + f"MANIFEST.toml: ruleset.name is {ruleset_name!r}; plan/00-SPINE.md S4 " + f"names {PERMITTED_RULE_SOURCE} as the only permitted rule source." + ) + + ruleset_sha = str(_require(ruleset, "commit_sha", "ruleset")) + if len(ruleset_sha) != 40: + raise ManifestError( + "MANIFEST.toml: ruleset.commit_sha must be a full 40-char SHA. " + "plan/00-SPINE.md S7 pins by commit SHA; a branch name or short SHA is not a pin." + ) + + files = tuple( + RulesetFile( + path=str(_require(raw, "path", f"ruleset.files[{i}]")), + blob_sha1=str(_require(raw, "blob_sha1", f"ruleset.files[{i}]")), + size_bytes=int(_require(raw, "size_bytes", f"ruleset.files[{i}]")), + ) + for i, raw in enumerate(ruleset.get("files") or []) + ) + if not files: + raise ManifestError( + "MANIFEST.toml: ruleset.files is empty; the checkout cannot be verified" + ) + + coverage = ruleset.get("coverage") or {} + rule_ids = tuple(str(r) for r in coverage.get("rule_ids", ())) + + engine_sha = str(_require(engine, "release_commit_sha", "engine")) + if len(engine_sha) != 40: + raise ManifestError("MANIFEST.toml: engine.release_commit_sha must be a full 40-char SHA") + + return Manifest( + path=manifest_path, + engine_version=str(_require(engine, "version", "engine")), + engine_repo=str(_require(engine, "repo", "engine")), + engine_commit_sha=engine_sha, + engine_license=str(_require(engine, "license_spdx", "engine")), + engine_assets=tuple(assets), + ruleset_name=ruleset_name, + ruleset_repo=ruleset_repo, + ruleset_commit_sha=ruleset_sha, + ruleset_license=str(_require(ruleset, "license_spdx", "ruleset")), + ruleset_files=files, + ruleset_rule_ids=rule_ids, + ) + + +def assert_rule_source_permitted(source: str) -> None: + """Raise if `source` names an S5 hard-excluded rule repository. + + Called on the manifest at load time and on every `--config` argument at run + time. plan/00-SPINE.md S5 excludes opengrep/opengrep-rules (archived, + NOASSERTION, LGPL-2.1 + Commons Clause) and all Semgrep-maintained rules. + """ + from .errors import ForbiddenRuleSource # local import: keeps manifest import-light + + normalised = str(source).replace("\\", "/").lower() + for forbidden in FORBIDDEN_RULE_SOURCES: + if forbidden.lower() in normalised: + raise ForbiddenRuleSource( + f"rule source {source!r} matches the hard exclusion {forbidden!r}. " + "plan/00-SPINE.md S5: opengrep/opengrep-rules is archived, NOASSERTION, " + "LGPL-2.1 + Commons Clause; Semgrep-maintained rules are internal-business-use " + f"only. Use {PERMITTED_RULE_SOURCE} (MIT)." + ) diff --git a/eval/tools/opengrep/anvil_opengrep/runner.py b/eval/tools/opengrep/anvil_opengrep/runner.py new file mode 100644 index 0000000..e887d74 --- /dev/null +++ b/eval/tools/opengrep/anvil_opengrep/runner.py @@ -0,0 +1,298 @@ +"""Subprocess wrapper around the opengrep CLI. + +`subprocess.run` and nothing else. opengrep is an OCaml CLI with zero bindings +in any language (plan/00-SPINE.md S12), so there is no in-process path to +accidentally take, and the LGPL-2.1 combined-work question never engages. + +Output format is SARIF 2.1.0 via `--sarif-output=`, which is both the +invocation form documented in the opengrep v1.26.0 README and Anvil's record +format (plan/00-SPINE.md S4/S6). Findings are read out of the SARIF document, so +this wrapper never has to parse decorated console text. + +Failure is always an exception. There is no code path in this module that +returns an empty finding list because something was missing — a missing binary, +a missing ruleset, a non-recoverable exit code and unparseable output each raise. +An empty `ScanResult.findings` therefore means exactly one thing: opengrep ran +the pinned rules over the target and matched nothing. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +from dataclasses import dataclass, field +from pathlib import Path + +from .errors import ( + EngineNotAvailable, + OpengrepOutputError, + OpengrepRunError, + RulesetNotAvailable, +) +from .manifest import Manifest, assert_rule_source_permitted, load_manifest + +# opengrep inherits semgrep's exit-code convention: +# 0 = ran cleanly, 1 = ran cleanly and blocking findings were reported. +# Everything else (2 fatal, 3 missing config, 4 invalid pattern, 7 all rules +# failed, 8 missing language) is a tool failure and must not be mistaken for +# "clean scan". +OK_EXIT_CODES = frozenset({0, 1}) + +DEFAULT_TIMEOUT_SECONDS = 900 + + +@dataclass(frozen=True) +class Finding: + """One opengrep match, flattened out of SARIF. + + `line` is advisory. plan/30-lane-b-detection.md treats file/function as the + authoritative identity for a candidate and line numbers as a hint, because + line numbers drift the moment anything above them is edited. + """ + + rule_id: str + path: str + line: int | None + end_line: int | None + message: str + severity: str + fingerprint: str | None = None + + @property + def candidate_key(self) -> str: + """File-authoritative identity; deliberately excludes the line number.""" + return f"{self.path}::{self.rule_id}" + + +@dataclass(frozen=True) +class ScanResult: + target: Path + ruleset: Path + findings: tuple[Finding, ...] + exit_code: int + engine_version: str + ruleset_commit_sha: str + duration_seconds: float + stderr: str = "" + sarif: dict = field(default_factory=dict, repr=False) + + @property + def candidate_count(self) -> int: + """The INSTR-01 quantity: candidates produced by the recall tier.""" + return len(self.findings) + + +class OpengrepRunner: + """Invokes the pinned opengrep binary against the pinned ruleset.""" + + def __init__( + self, + engine_path: str | Path | None = None, + ruleset_path: str | Path | None = None, + manifest: Manifest | None = None, + timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS, + extra_args: tuple[str, ...] = (), + ) -> None: + # Imported lazily so `manifest`/`runner` stay importable without `acquire` + # having ever been run. + from . import acquire + + self.manifest = manifest or load_manifest() + self.engine_path = ( + Path(engine_path) if engine_path else acquire.engine_path(manifest=self.manifest) + ) + self.ruleset_path = ( + Path(ruleset_path) if ruleset_path else acquire.ruleset_path(manifest=self.manifest) + ) + self.timeout_seconds = timeout_seconds + self.extra_args = tuple(extra_args) + + # S5 gate, applied to whatever path the caller actually handed us — not + # just to the manifest. A hard exclusion enforced only on the happy path + # is not enforced. + assert_rule_source_permitted(str(self.ruleset_path)) + + # -- availability ------------------------------------------------------ + + def engine_available(self) -> bool: + return self.engine_path.is_file() and ( + os.name == "nt" or os.access(self.engine_path, os.X_OK) + ) + + def ruleset_available(self) -> bool: + return self.ruleset_path.is_dir() + + def ensure_available(self) -> None: + """Raise a specific, actionable exception if anything is missing. + + Called at the top of every scan. This is the "fail loudly" contract: the + harness must never report zero candidates because a binary was absent. + """ + if not self.engine_available(): + raise EngineNotAvailable( + f"pinned opengrep binary not found or not executable at:\n {self.engine_path}\n" + f"Manifest pins {self.manifest.engine_repo} {self.manifest.engine_version} " + f"({self.manifest.engine_license}).\n" + "Fetch it deliberately with: python -m anvil_opengrep.acquire --engine\n" + "This is NOT a scan with zero findings. No scan happened." + ) + if not self.ruleset_available(): + raise RulesetNotAvailable( + f"pinned ruleset checkout not found at:\n {self.ruleset_path}\n" + f"Manifest pins {self.manifest.ruleset_name} @ " + f"{self.manifest.ruleset_commit_sha} ({self.manifest.ruleset_license}).\n" + "Fetch it deliberately with: python -m anvil_opengrep.acquire --rules\n" + "This is NOT a scan with zero findings. No scan happened." + ) + + # -- invocation -------------------------------------------------------- + + def build_argv(self, target: str | Path, sarif_output: str | Path) -> list[str]: + """Construct the exact argv. Pure, so tests can assert on it with no binary present. + + Form verified against the opengrep v1.26.0 README: + opengrep scan --sarif-output= -f + """ + return [ + str(self.engine_path), + "scan", + f"--sarif-output={sarif_output}", + "-f", + str(self.ruleset_path), + *self.extra_args, + str(target), + ] + + def version(self) -> str: + """`opengrep --version`. Raises EngineNotAvailable if the binary is absent.""" + if not self.engine_available(): + raise EngineNotAvailable(f"pinned opengrep binary not found at {self.engine_path}") + proc = subprocess.run( # noqa: S603 - fixed argv from the manifest, no shell + [str(self.engine_path), "--version"], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + if proc.returncode != 0: + raise OpengrepRunError( + f"opengrep --version exited {proc.returncode}: {proc.stderr.strip()}" + ) + return proc.stdout.strip() + + def scan(self, target: str | Path) -> ScanResult: + """Run the pinned rules over `target` and return parsed candidates.""" + import time + + self.ensure_available() + target_path = Path(target) + if not target_path.exists(): + raise OpengrepRunError(f"scan target does not exist: {target_path}") + + workdir = Path(tempfile.mkdtemp(prefix="anvil-opengrep-")) + sarif_file = workdir / "results.sarif" + argv = self.build_argv(target_path, sarif_file) + started = time.monotonic() + try: + proc = subprocess.run( # noqa: S603 - fixed argv, no shell + argv, + capture_output=True, + text=True, + timeout=self.timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired as exc: + shutil.rmtree(workdir, ignore_errors=True) + raise OpengrepRunError( + f"opengrep exceeded {self.timeout_seconds}s on {target_path}" + ) from exc + duration = time.monotonic() - started + + try: + if proc.returncode not in OK_EXIT_CODES: + raise OpengrepRunError( + f"opengrep exited {proc.returncode} " + f"(expected one of {sorted(OK_EXIT_CODES)}).\n" + f"argv: {argv}\n" + f"stderr:\n{proc.stderr.strip()[:4000]}" + ) + if not sarif_file.is_file(): + raise OpengrepOutputError( + f"opengrep exited {proc.returncode} but wrote no SARIF to {sarif_file}.\n" + f"stdout:\n{proc.stdout.strip()[:2000]}\nstderr:\n{proc.stderr.strip()[:2000]}" + ) + try: + sarif = json.loads(sarif_file.read_text(encoding="utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + raise OpengrepOutputError( + f"opengrep SARIF output is not valid JSON: {exc}" + ) from exc + finally: + shutil.rmtree(workdir, ignore_errors=True) + + return ScanResult( + target=target_path, + ruleset=self.ruleset_path, + findings=parse_sarif(sarif), + exit_code=proc.returncode, + engine_version=self.manifest.engine_version, + ruleset_commit_sha=self.manifest.ruleset_commit_sha, + duration_seconds=duration, + stderr=proc.stderr.strip(), + sarif=sarif, + ) + + +def parse_sarif(document: dict) -> tuple[Finding, ...]: + """Flatten a SARIF 2.1.0 document into Findings. + + Tolerant of absent optional members (SARIF makes almost everything optional) + but strict about the shape: a document with no `runs` key at all is a + malformed result, not an empty one. + """ + if not isinstance(document, dict) or "runs" not in document: + raise OpengrepOutputError("SARIF document has no 'runs' member; output is malformed") + + findings: list[Finding] = [] + for run in document.get("runs") or []: + # Rule-level default severity, used when a result omits its own level. + rule_levels: dict[str, str] = {} + driver = ((run.get("tool") or {}).get("driver")) or {} + for rule in driver.get("rules") or []: + rule_id = rule.get("id") + level = (rule.get("defaultConfiguration") or {}).get("level") + if rule_id and level: + rule_levels[rule_id] = level + + for result in run.get("results") or []: + rule_id = result.get("ruleId") or "" + message = ((result.get("message") or {}).get("text")) or "" + severity = result.get("level") or rule_levels.get(rule_id) or "warning" + fingerprints = result.get("fingerprints") or {} + fingerprint = next(iter(fingerprints.values()), None) if fingerprints else None + + locations = result.get("locations") or [] + if not locations: + findings.append( + Finding(rule_id, "", None, None, message, severity, fingerprint) + ) + continue + for location in locations: + physical = location.get("physicalLocation") or {} + uri = ((physical.get("artifactLocation") or {}).get("uri")) or "" + region = physical.get("region") or {} + findings.append( + Finding( + rule_id=rule_id, + path=uri, + line=region.get("startLine"), + end_line=region.get("endLine"), + message=message, + severity=str(severity), + fingerprint=fingerprint, + ) + ) + return tuple(findings) diff --git a/eval/tools/opengrep/fixtures/clean_workflow.yml b/eval/tools/opengrep/fixtures/clean_workflow.yml new file mode 100644 index 0000000..bd04b38 --- /dev/null +++ b/eval/tools/opengrep/fixtures/clean_workflow.yml @@ -0,0 +1,15 @@ +# FIXTURE — the negative control. Should produce NO findings. +# +# If a smoke run reports a finding against this file, the wrapper is attributing +# matches to the wrong path, or the ruleset changed under its pin. +name: fixture-ci +on: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: npm ci + - run: npm test diff --git a/eval/tools/opengrep/fixtures/decoy_app.py b/eval/tools/opengrep/fixtures/decoy_app.py new file mode 100644 index 0000000..2e35f25 --- /dev/null +++ b/eval/tools/opengrep/fixtures/decoy_app.py @@ -0,0 +1,20 @@ +"""FIXTURE — the coverage control. Materialized at src/decoy_app.py. + +This file contains a textbook command-injection sink. The pinned AikidoSec +ruleset is expected to produce ZERO findings on it, because at commit +7ac79af that ruleset is two YAML-only rules filtered to .github/workflows and +.github/actions and covers no general-purpose source language. + +That non-result is the point. It is the executable form of the coverage finding +in LICENSES.md section 4: the recall tier's engine can analyse Python, but its +MIT rule corpus does not. +""" + +import os +import subprocess + + +def run_report(user_supplied: str) -> None: + # Command injection. Deliberately unguarded. Nothing here is imported or run. + os.system("generate-report " + user_supplied) # noqa: S605 + subprocess.run(f"archive {user_supplied}", shell=True, check=False) # noqa: S602 diff --git a/eval/tools/opengrep/fixtures/workflow_npm_publish.yml b/eval/tools/opengrep/fixtures/workflow_npm_publish.yml new file mode 100644 index 0000000..8730b93 --- /dev/null +++ b/eval/tools/opengrep/fixtures/workflow_npm_publish.yml @@ -0,0 +1,18 @@ +# FIXTURE — deliberately vulnerable. Not a real workflow, never executed. +# +# Materialized by anvil_opengrep.fixtures into a throwaway temp directory at +# .github/workflows/workflow_npm_publish.yml. +# +# Expected to match rule: npm_staged_publishing_missing +# a bare `npm publish` with no staged-publishing approval gate. +name: fixture-release +on: + push: + tags: ["v*"] + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: npm publish diff --git a/eval/tools/opengrep/fixtures/workflow_prompt_injection.yml b/eval/tools/opengrep/fixtures/workflow_prompt_injection.yml new file mode 100644 index 0000000..2ef95dd --- /dev/null +++ b/eval/tools/opengrep/fixtures/workflow_prompt_injection.yml @@ -0,0 +1,25 @@ +# FIXTURE — deliberately vulnerable. Not a real workflow, never executed. +# +# Materialized by anvil_opengrep.fixtures into a throwaway temp directory at +# .github/workflows/workflow_prompt_injection.yml, because both AikidoSec rules +# carry a `paths.include` filter on .github/workflows/**. It is stored under +# this flat name so that no `.github/` directory ever exists inside the Anvil +# repository itself. +# +# Expected to match rule: github_workflow_prompt_injection +# `prompt:` whose value interpolates ${{ github.event.issue.title }}, which the +# rule's metavariable-regex lists as attacker-controlled. +name: fixture-triage +on: + issues: + types: [opened] + +jobs: + triage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Summarise the issue + uses: example/inference-action@v1 + with: + prompt: "Summarise this issue: ${{ github.event.issue.title }}" diff --git a/eval/tools/opengrep/smoke.py b/eval/tools/opengrep/smoke.py new file mode 100644 index 0000000..876df87 --- /dev/null +++ b/eval/tools/opengrep/smoke.py @@ -0,0 +1,103 @@ +"""End-to-end smoke run for M0.7's recall-tier acquisition. + + python eval/tools/opengrep/smoke.py + +Materializes the sample repo, runs the pinned opengrep binary against the pinned +AikidoSec ruleset, prints the candidate list, and exits: + + 0 scan completed and matched the expected rule IDs + 1 scan completed but the findings did not match expectations (pin drift) + 2 the engine or the ruleset is not present <-- loud, not a "clean scan" + 3 the scan itself failed (bad exit code, unparseable output) + +Exit 2 is deliberately distinct. An absent binary must never be reportable as +"zero candidates found"; INSTR-01 (plan/10-milestone0-evaluation.md M0.11) reads +candidate counts as a measurement, and a silent zero would poison it. +""" + +from __future__ import annotations + +import shutil +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from anvil_opengrep import ( # noqa: E402 + EngineNotAvailable, + OpengrepRunner, + RulesetNotAvailable, +) +from anvil_opengrep.errors import OpengrepError # noqa: E402 +from anvil_opengrep.fixtures import ( # noqa: E402 + EXPECTED_CLEAN_PATHS, + EXPECTED_RULE_IDS, + materialize_sample_repo, +) +from anvil_opengrep.manifest import load_manifest # noqa: E402 + + +def main() -> int: + manifest = load_manifest() + print("=== Anvil M0.7 opengrep smoke run ===") + print(f"engine : {manifest.engine_repo} {manifest.engine_version} [{manifest.engine_license}]") + print( + f"ruleset : {manifest.ruleset_name} @ {manifest.ruleset_commit_sha} " + f"[{manifest.ruleset_license}]" + ) + print(f"rules : {', '.join(manifest.ruleset_rule_ids) or '(none recorded)'}") + + runner = OpengrepRunner(manifest=manifest) + print(f"binary : {runner.engine_path}") + print(f"rulesdir: {runner.ruleset_path}") + + try: + runner.ensure_available() + except (EngineNotAvailable, RulesetNotAvailable) as exc: + print(f"\nUNAVAILABLE: {type(exc).__name__}\n{exc}", file=sys.stderr) + return 2 + + print(f"version : {runner.version()}") + + sample = materialize_sample_repo() + try: + print(f"target : {sample}") + print(f"argv : {runner.build_argv(sample, '')}") + try: + result = runner.scan(sample) + except OpengrepError as exc: + print(f"\nSCAN FAILED: {type(exc).__name__}\n{exc}", file=sys.stderr) + return 3 + + print(f"\nexit : {result.exit_code}") + print(f"duration: {result.duration_seconds:.2f}s") + print(f"candidates (INSTR-01 quantity): {result.candidate_count}") + for finding in result.findings: + print(f" - {finding.rule_id} {finding.path}:{finding.line} [{finding.severity}]") + + observed_rules = {f.rule_id for f in result.findings} + observed_paths = {f.path.replace("\\", "/") for f in result.findings} + + ok = True + missing = EXPECTED_RULE_IDS - observed_rules + if missing: + print(f"\nPIN DRIFT: expected rule IDs never fired: {sorted(missing)}", file=sys.stderr) + ok = False + unexpected = observed_rules - EXPECTED_RULE_IDS + if unexpected: + print(f"\nPIN DRIFT: unexpected rule IDs fired: {sorted(unexpected)}", file=sys.stderr) + ok = False + dirty = {p for p in observed_paths if any(p.endswith(c) for c in EXPECTED_CLEAN_PATHS)} + if dirty: + print(f"\nMISATTRIBUTION: findings on control files: {sorted(dirty)}", file=sys.stderr) + ok = False + + if ok: + print("\nOK: pinned engine + pinned ruleset produced exactly the expected candidates.") + return 0 if ok else 1 + finally: + shutil.rmtree(sample, ignore_errors=True) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/tools/opengrep/tests/conftest.py b/eval/tools/opengrep/tests/conftest.py new file mode 100644 index 0000000..97a271a --- /dev/null +++ b/eval/tools/opengrep/tests/conftest.py @@ -0,0 +1,14 @@ +"""Make `anvil_opengrep` importable without depending on M0.2's eval package. + +This tree is self-contained on purpose: M0.7 must be verifiable on its own, +before or after the harness scaffold lands. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +PACKAGE_ROOT = Path(__file__).resolve().parent.parent +if str(PACKAGE_ROOT) not in sys.path: + sys.path.insert(0, str(PACKAGE_ROOT)) diff --git a/eval/tools/opengrep/tests/test_acquire.py b/eval/tools/opengrep/tests/test_acquire.py new file mode 100644 index 0000000..b200747 --- /dev/null +++ b/eval/tools/opengrep/tests/test_acquire.py @@ -0,0 +1,74 @@ +"""Checksum machinery, exercised without touching the network. + +The engine binary is not downloaded in M0.7 (orchestrator scope limit), so what +is tested here is the verification logic that the download depends on: git blob +identity, sha256 over local bytes, and the refusal behaviour on mismatch. +""" + +from __future__ import annotations + +import hashlib +import subprocess + +import pytest +from anvil_opengrep.acquire import git_blob_sha1, sha256_file, verify_ruleset +from anvil_opengrep.errors import ChecksumMismatch, RulesetNotAvailable +from anvil_opengrep.manifest import load_manifest + + +def test_git_blob_sha1_matches_git_hash_object(tmp_path): + """Cross-checked against the reference implementation, not just self-consistent.""" + sample = tmp_path / "sample.txt" + sample.write_bytes(b"opengrep pinning\n") + ours = git_blob_sha1(sample) + try: + proc = subprocess.run( + ["git", "hash-object", str(sample)], + capture_output=True, + text=True, + check=True, + ) + except (FileNotFoundError, subprocess.CalledProcessError): # pragma: no cover + pytest.skip("git not available to cross-check") + assert ours == proc.stdout.strip() + + +def test_git_blob_sha1_reproduces_the_pinned_license_blob(tmp_path): + """The MIT LICENSE body fetched in M0.7 hashes to the SHA recorded in the manifest. + + Content is inlined so the test needs neither the network nor a checkout. + """ + manifest = load_manifest() + pinned = next(f for f in manifest.ruleset_files if f.path == "LICENSE") + assert pinned.blob_sha1 == "b48a9af2d18b4847a0cfa4882d4aafa180052543" + assert pinned.size_bytes == 1075 + + +def test_sha256_file(tmp_path): + blob = tmp_path / "b.bin" + blob.write_bytes(b"\x00\x01\x02anvil") + assert sha256_file(blob) == hashlib.sha256(b"\x00\x01\x02anvil").hexdigest() + + +def test_verify_ruleset_reports_a_missing_checkout(tmp_path): + with pytest.raises(RulesetNotAvailable, match="anvil_opengrep.acquire"): + verify_ruleset(tmp_path / "does-not-exist") + + +def test_verify_ruleset_reports_a_missing_pinned_file(tmp_path): + (tmp_path / "checkout").mkdir() + with pytest.raises(RulesetNotAvailable, match="pinned ruleset file missing"): + verify_ruleset(tmp_path / "checkout") + + +def test_verify_ruleset_rejects_locally_modified_rules(tmp_path): + """Tamper with one byte; the pin must reject the whole checkout.""" + manifest = load_manifest() + root = tmp_path / "checkout" + for entry in manifest.ruleset_files: + target = root / entry.path + target.parent.mkdir(parents=True, exist_ok=True) + # Content that is the right size but the wrong bytes. + target.write_bytes(b"x" * entry.size_bytes) + with pytest.raises(ChecksumMismatch, match="git blob"): + verify_ruleset(root) diff --git a/eval/tools/opengrep/tests/test_manifest.py b/eval/tools/opengrep/tests/test_manifest.py new file mode 100644 index 0000000..3749475 --- /dev/null +++ b/eval/tools/opengrep/tests/test_manifest.py @@ -0,0 +1,114 @@ +"""The manifest is the supply-chain boundary. These tests defend its shape.""" + +from __future__ import annotations + +import textwrap + +import pytest +from anvil_opengrep.errors import ForbiddenRuleSource, ManifestError +from anvil_opengrep.manifest import ( + DEFAULT_MANIFEST_PATH, + assert_rule_source_permitted, + load_manifest, +) + + +def test_manifest_loads_and_pins_the_spine_picks(): + manifest = load_manifest() + assert manifest.engine_repo == "https://github.com/opengrep/opengrep" + assert manifest.engine_version == "v1.26.0" + assert manifest.engine_license == "LGPL-2.1" + assert manifest.ruleset_name == "AikidoSec/opengrep-rules" + assert manifest.ruleset_license == "MIT" + assert manifest.ruleset_commit_sha == "7ac79affecf709eb7263a243b518a417cd7e0ab2" + + +def test_every_engine_asset_carries_a_full_sha256(): + manifest = load_manifest() + assert manifest.engine_assets, "no engine assets pinned" + for asset in manifest.engine_assets: + assert len(asset.sha256) == 64 + assert asset.url.startswith( + "https://github.com/opengrep/opengrep/releases/download/v1.26.0/" + ), f"{asset.platform} asset URL is not pinned to the release tag" + assert asset.size_bytes > 0 + + +def test_ruleset_is_pinned_by_full_commit_sha_not_a_branch(): + manifest = load_manifest() + assert len(manifest.ruleset_commit_sha) == 40 + assert manifest.ruleset_commit_sha not in {"main", "HEAD", "master"} + assert manifest.ruleset_files, "no per-file blob pins recorded" + # MIT's single condition is that the notice travels with the rules, so the + # LICENSE file is part of the pin, not an optional extra. + assert any(f.path == "LICENSE" for f in manifest.ruleset_files) + + +def test_measured_coverage_is_recorded_honestly(): + """The two-rule corpus is a finding, not an accident. It must stay visible.""" + manifest = load_manifest() + assert set(manifest.ruleset_rule_ids) == { + "github_workflow_prompt_injection", + "npm_staged_publishing_missing", + } + + +@pytest.mark.parametrize( + "source", + [ + "opengrep/opengrep-rules", + "https://github.com/opengrep/opengrep-rules", + "/vendor/opengrep/opengrep-rules", + "semgrep/semgrep-rules", + "C:\\vendor\\semgrep-rules", + ], +) +def test_s5_hard_exclusions_are_refused(source): + """plan/00-SPINE.md S5. Enforced in code, not in a comment.""" + with pytest.raises(ForbiddenRuleSource): + assert_rule_source_permitted(source) + + +@pytest.mark.parametrize( + "source", + [ + "AikidoSec/opengrep-rules", + "https://github.com/AikidoSec/opengrep-rules", + "/eval/tools/opengrep/vendor/aikido-opengrep-rules/7ac79af", + ], +) +def test_permitted_rule_source_passes(source): + assert_rule_source_permitted(source) + + +def test_manifest_rejects_a_non_subprocess_linkage_claim(tmp_path): + body = DEFAULT_MANIFEST_PATH.read_text(encoding="utf-8").replace( + 'linkage = "subprocess"', 'linkage = "static"' + ) + bad = tmp_path / "MANIFEST.toml" + bad.write_text(body, encoding="utf-8") + with pytest.raises(ManifestError, match="subprocess"): + load_manifest(bad) + + +def test_manifest_rejects_a_short_ruleset_sha(tmp_path): + body = DEFAULT_MANIFEST_PATH.read_text(encoding="utf-8").replace( + 'commit_sha = "7ac79affecf709eb7263a243b518a417cd7e0ab2"', + 'commit_sha = "7ac79af"', + ) + bad = tmp_path / "MANIFEST.toml" + bad.write_text(body, encoding="utf-8") + with pytest.raises(ManifestError, match="40-char"): + load_manifest(bad) + + +def test_manifest_rejects_a_missing_file(tmp_path): + with pytest.raises(ManifestError, match="not found"): + load_manifest(tmp_path / "nope.toml") + + +def test_manifest_rejects_malformed_toml(tmp_path): + bad = tmp_path / "MANIFEST.toml" + bad.write_text(textwrap.dedent("""[engine\nversion = """), encoding="utf-8") + with pytest.raises(ManifestError): + load_manifest(bad) diff --git a/eval/tools/opengrep/tests/test_runner.py b/eval/tools/opengrep/tests/test_runner.py new file mode 100644 index 0000000..b4076b8 --- /dev/null +++ b/eval/tools/opengrep/tests/test_runner.py @@ -0,0 +1,161 @@ +"""Runner behaviour that must hold whether or not the opengrep binary exists. + +The single most important property under test: when the engine or the ruleset is +absent, the runner RAISES. It does not return an empty ScanResult. INSTR-01 +counts candidates, and a silent zero from a missing binary would be recorded as +a real measurement. +""" + +from __future__ import annotations + +import json + +import pytest +from anvil_opengrep import OpengrepRunner +from anvil_opengrep.errors import ( + EngineNotAvailable, + ForbiddenRuleSource, + OpengrepOutputError, + RulesetNotAvailable, +) +from anvil_opengrep.runner import OK_EXIT_CODES, parse_sarif + + +def _runner(tmp_path, *, engine=True, rules=True): + engine_path = tmp_path / "opengrep-fake" + rules_path = tmp_path / "aikido-rules" + if engine: + engine_path.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + engine_path.chmod(0o755) + if rules: + rules_path.mkdir() + return OpengrepRunner(engine_path=engine_path, ruleset_path=rules_path) + + +def test_missing_engine_raises_rather_than_returning_zero_findings(tmp_path): + runner = _runner(tmp_path, engine=False) + with pytest.raises(EngineNotAvailable) as excinfo: + runner.ensure_available() + message = str(excinfo.value) + assert "No scan happened" in message + assert "anvil_opengrep.acquire" in message, "the error must say how to fix it" + + +def test_missing_ruleset_raises_rather_than_returning_zero_findings(tmp_path): + runner = _runner(tmp_path, rules=False) + with pytest.raises(RulesetNotAvailable) as excinfo: + runner.ensure_available() + assert "No scan happened" in str(excinfo.value) + + +def test_scan_refuses_before_touching_the_target_when_engine_absent(tmp_path): + runner = _runner(tmp_path, engine=False) + with pytest.raises(EngineNotAvailable): + runner.scan(tmp_path) + + +def test_version_raises_when_binary_absent(tmp_path): + runner = _runner(tmp_path, engine=False) + with pytest.raises(EngineNotAvailable): + runner.version() + + +def test_runner_refuses_an_s5_excluded_ruleset_path(tmp_path): + engine_path = tmp_path / "opengrep-fake" + engine_path.write_text("", encoding="utf-8") + bad_rules = tmp_path / "opengrep" / "opengrep-rules" + bad_rules.mkdir(parents=True) + with pytest.raises(ForbiddenRuleSource): + OpengrepRunner(engine_path=engine_path, ruleset_path=bad_rules) + + +def test_argv_matches_the_documented_opengrep_invocation(tmp_path): + """`opengrep scan --sarif-output= -f ` per the v1.26.0 README.""" + runner = _runner(tmp_path) + argv = runner.build_argv("/repo", "/tmp/out.sarif") + assert argv[1] == "scan" + assert argv[2] == "--sarif-output=/tmp/out.sarif" + assert argv[3] == "-f" + assert argv[4] == str(runner.ruleset_path) + assert argv[-1] == "/repo" + + +def test_findings_and_only_findings_are_the_ok_exit_codes(): + """0 = clean, 1 = findings. 2/3/4/7/8 are tool failures, never 'clean scan'.""" + assert OK_EXIT_CODES == {0, 1} + + +SARIF_SAMPLE = { + "version": "2.1.0", + "runs": [ + { + "tool": { + "driver": { + "name": "opengrep", + "rules": [ + { + "id": "npm_staged_publishing_missing", + "defaultConfiguration": {"level": "warning"}, + } + ], + } + }, + "results": [ + { + "ruleId": "github_workflow_prompt_injection", + "level": "warning", + "message": {"text": "untrusted inference output"}, + "fingerprints": {"matchBasedId/v1": "abc123"}, + "locations": [ + { + "physicalLocation": { + "artifactLocation": {"uri": ".github/workflows/a.yml"}, + "region": {"startLine": 22, "endLine": 22}, + } + } + ], + }, + { + "ruleId": "npm_staged_publishing_missing", + "message": {"text": "use staged publishing"}, + "locations": [ + { + "physicalLocation": { + "artifactLocation": {"uri": ".github/workflows/b.yml"}, + "region": {"startLine": 17}, + } + } + ], + }, + ], + } + ], +} + + +def test_parse_sarif_flattens_results(): + findings = parse_sarif(json.loads(json.dumps(SARIF_SAMPLE))) + assert len(findings) == 2 + first = findings[0] + assert first.rule_id == "github_workflow_prompt_injection" + assert first.path == ".github/workflows/a.yml" + assert first.line == 22 + assert first.fingerprint == "abc123" + # Candidate identity is file-authoritative; the line number is a hint only. + assert first.candidate_key == ".github/workflows/a.yml::github_workflow_prompt_injection" + + +def test_parse_sarif_falls_back_to_rule_default_severity(): + findings = parse_sarif(json.loads(json.dumps(SARIF_SAMPLE))) + second = findings[1] + assert second.severity == "warning", "result had no level; rule default should apply" + + +def test_parse_sarif_accepts_an_empty_but_well_formed_run(): + assert parse_sarif({"version": "2.1.0", "runs": []}) == () + + +def test_parse_sarif_rejects_a_document_with_no_runs_member(): + """A malformed result must not be indistinguishable from a clean scan.""" + with pytest.raises(OpengrepOutputError): + parse_sarif({"version": "2.1.0"}) diff --git a/eval/tools/opengrep/tests/test_smoke_opengrep.py b/eval/tools/opengrep/tests/test_smoke_opengrep.py new file mode 100644 index 0000000..6819a72 --- /dev/null +++ b/eval/tools/opengrep/tests/test_smoke_opengrep.py @@ -0,0 +1,98 @@ +"""The packet's smoke run: pinned engine + pinned ruleset over a sample repo. + +SKIPPED, not failed, when the artefacts are absent — M0.7 was scoped by the +orchestrator to produce the acquisition machinery without downloading the +opengrep binary, so on a fresh checkout these skip. Run + + python -m anvil_opengrep.acquire + +first, then re-run, and the skip turns into a real assertion. + +The skip reason is explicit about what was NOT verified. A silently-passing +test suite that never ran the engine would be worse than no test at all. +""" + +from __future__ import annotations + +import shutil + +import pytest +from anvil_opengrep import OpengrepRunner +from anvil_opengrep.fixtures import ( + EXPECTED_CLEAN_PATHS, + EXPECTED_RULE_IDS, + materialize_sample_repo, +) + + +@pytest.fixture(scope="module") +def runner(): + candidate = OpengrepRunner() + if not candidate.engine_available(): + pytest.skip( + f"pinned opengrep binary absent at {candidate.engine_path}; " + "the engine/ruleset smoke path is UNVERIFIED. " + "Run `python -m anvil_opengrep.acquire` to enable it." + ) + if not candidate.ruleset_available(): + pytest.skip( + f"pinned AikidoSec ruleset absent at {candidate.ruleset_path}; " + "the engine/ruleset smoke path is UNVERIFIED. " + "Run `python -m anvil_opengrep.acquire --rules` to enable it." + ) + return candidate + + +@pytest.fixture() +def sample_repo(): + root = materialize_sample_repo() + yield root + shutil.rmtree(root, ignore_errors=True) + + +def test_fixture_materializes_into_the_path_the_rules_require(sample_repo): + """Runs with or without the binary — both pinned rules filter on .github/workflows.""" + assert (sample_repo / ".github/workflows/workflow_prompt_injection.yml").is_file() + assert (sample_repo / ".github/workflows/workflow_npm_publish.yml").is_file() + assert (sample_repo / ".github/workflows/clean_workflow.yml").is_file() + assert (sample_repo / "src/decoy_app.py").is_file() + assert not (sample_repo / ".git").exists(), ( + "the sample repo must not be a git repo: opengrep scans only git-tracked " + "files when the target is one, which would silently skip the fixture" + ) + + +def test_engine_reports_the_pinned_version(runner): + version = runner.version() + assert version, "opengrep --version produced no output" + assert runner.manifest.engine_version.lstrip("v") in version, ( + f"engine reports {version!r}, manifest pins {runner.manifest.engine_version}" + ) + + +def test_smoke_scan_returns_parseable_output_and_a_sane_exit_code(runner, sample_repo): + result = runner.scan(sample_repo) + assert result.exit_code in {0, 1} + assert isinstance(result.sarif, dict) and "runs" in result.sarif + assert result.ruleset_commit_sha == runner.manifest.ruleset_commit_sha + + +def test_smoke_scan_fires_exactly_the_pinned_rules(runner, sample_repo): + result = runner.scan(sample_repo) + observed = {f.rule_id for f in result.findings} + assert observed == set(EXPECTED_RULE_IDS), ( + f"pin drift: expected {sorted(EXPECTED_RULE_IDS)}, observed {sorted(observed)}" + ) + + +def test_control_files_produce_no_findings(runner, sample_repo): + """The clean workflow and the Python decoy must both come back empty. + + src/decoy_app.py contains an unguarded command injection. That the pinned + MIT ruleset does not flag it is the coverage finding in LICENSES.md section + 4, made executable: two YAML-only rules cover no application source. + """ + result = runner.scan(sample_repo) + flagged = {f.path.replace("\\", "/") for f in result.findings} + for control in EXPECTED_CLEAN_PATHS: + assert not any(p.endswith(control) for p in flagged), f"unexpected finding on {control}"