Phase 1: M0 experiment register, eval harness scaffold, opengrep acquisition

.github/workflows/ci.yml

@@ -140,6 +140,50 @@ jobs:
             exit 1
           fi
 
+  # plan/00-SPINE.md S12 permits Python in exactly three places, and the
+  # evaluation harness is one of them. eval/ is a pure-Python tree.
+  #
+  # The register this job validates is the artifact plan step M0.18 reads to
+  # decide whether Anvil's detection-model tier exists at all, so the property
+  # that matters is narrow and absolute: a row whose experiment has not run must
+  # never be readable as a pass. The schema encodes that; this job is what makes
+  # anything actually run the schema. It shipped with nothing in the repository
+  # checking one against the other, and a schema nobody runs is a comment.
+  eval-harness:
+    name: Eval harness (Python)
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: eval
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      # [dev] carries pytest/ruff. Core deps are deliberately minimal -- the
+      # heavy [stats] and [models] extras are pulled by the packets that need
+      # them (M0.6, M0.10, M0.15), not here.
+      - name: Install
+        run: python -m pip install --upgrade pip && python -m pip install -e ".[dev]"
+
+      - name: Lint
+        run: python -m ruff check .
+
+      - name: Tests, including the register mutation suite
+        run: python -m pytest tests/ -q
+
+      # A skipped test is a legitimate outcome here -- M0.7's smoke path is
+      # UNVERIFIED by design because nothing downloads the opengrep engine in
+      # CI. But a suite that silently degrades to all-skips would still report
+      # green, so assert the register suite actually ran.
+      - name: The register mutation suite really ran
+        run: |
+          out=$(python -m pytest tests/test_register_schema.py -q 2>&1)
+          echo "$out"
+          echo "$out" | grep -qE '[0-9]+ passed' || {
+            echo "::error::register schema suite reported no passing tests"; exit 1; }
+
   # NOTE: there is deliberately no "plan" job here. The planning workspace
   # (plan/) and its verification tooling (tools/) are local-only and gitignored,
   # same posture as research/ -- they are how Anvil was designed and verified,

eval/.gitignore

@@ -0,0 +1,81 @@
+# eval/.gitignore — Anvil evaluation harness (M0.2)
+#
+# The repository-root .gitignore already covers the generic Python noise
+# (__pycache__/, *.egg-info/, .venv/) and blanket-ignores `models/`,
+# `*.gguf` and `*.safetensors`. This file adds the eval-tree-specific rules
+# for model and dataset caches, and re-includes the provenance metadata that
+# the Milestone 0 register depends on.
+#
+# Deeper .gitignore files take precedence over shallower ones, so the
+# re-inclusions below override the root file's `models/` rule for eval/models.
+
+# --- Dataset payloads (M0.3 PrimeVul, M0.4 ARVO, M0.5 CWE-Bench-Java) -------
+# Corpora are acquired locally and never committed or re-hosted:
+# PrimeVul is Google-Drive-gated (M0.3 forbidden actions) and ARVO /
+# CWE-Bench-Java are pulled from their upstream repos at a pinned SHA.
+data/**
+# ...but keep the directory skeleton, the MANIFEST files that record the
+# acquisition-date snapshot and pinned SHAs, and any prose notes.
+!data/**/
+!data/**/MANIFEST
+!data/**/MANIFEST.*
+!data/**/*.md
+!data/**/.gitkeep
+
+# --- Model artifacts (M0.6) -------------------------------------------------
+# Weights are never committed. Only the pinned-download manifest, the exact HF
+# revision SHA, the archived LICENSE text (S8 compliance mechanics) and the
+# checksums are tracked.
+!models/
+models/**
+!models/**/
+!models/**/MANIFEST
+!models/**/MANIFEST.*
+!models/**/*.md
+!models/**/*.json
+!models/**/*.txt
+!models/**/*.sha256
+!models/**/LICENSE*
+!models/**/.gitkeep
+# Belt and braces: never track a weight file even if a rule above widens.
+models/**/*.gguf
+models/**/*.safetensors
+models/**/*.onnx
+models/**/*.onnx_data
+models/**/*.bin
+models/**/*.pt
+models/**/*.pth
+
+# --- Tool binaries (M0.7 opengrep engine + pinned ruleset) ------------------
+# The repository-root .gitignore blanket-ignores `tools/` (unanchored, so it
+# matches eval/tools/ too). That rule is meant for the root-level planning
+# tooling; eval/tools/ is M0.7's write scope and its pin records must be
+# tracked, so re-include the directory here and exclude only the payloads.
+!tools/
+tools/**/bin/
+tools/**/*.exe
+tools/**/*.tar.gz
+tools/**/*.zip
+tools/**/*.tgz
+tools/**/*.whl
+
+# --- Hugging Face / dataset download caches --------------------------------
+.hf_cache/
+.cache/
+hf_home/
+transformers_cache/
+datasets_cache/
+
+# --- Local run scratch ------------------------------------------------------
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.coverage
+.coverage.*
+htmlcov/
+scratch/
+*.log
+
+# NOTE: eval/results/ is deliberately NOT ignored — the Milestone 0 exit
+# criteria require every register row's `artifact_path` to point at a real,
+# committed eval/results/<ID>.json file.

eval/pyproject.toml

@@ -0,0 +1,81 @@
+[build-system]
+requires = ["setuptools>=77.0.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "anvil-eval"
+version = "0.1.0"
+description = "Anvil Milestone 0 evaluation harness — experiment register, corpora loaders, and experiment runners."
+requires-python = ">=3.11"
+license = "Apache-2.0"
+authors = [{ name = "Anvil maintainers" }]
+keywords = ["vulnerability", "evaluation", "harness", "anvil", "milestone-0"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Security",
+  "Private :: Do Not Upload",
+]
+
+# Core runtime dependencies. Kept deliberately small so that
+# `pip install -e eval/` resolves quickly in a fresh virtualenv with no
+# network access beyond the package index (M0.2 stop condition).
+# Anything heavy, platform-specific, or only needed by one later M0 step
+# lives in an optional-dependency group below.
+dependencies = [
+  "jsonschema>=4.23",  # M0.1/M0.17 register validation against eval/schema/register.schema.json
+  "pyyaml>=6.0.2",     # eval/register.yaml read/write
+  "numpy>=2.1",        # bootstrap confidence intervals (EXP-01, EXP-02)
+  "requests>=2.32",    # pinned-artifact fetches, S12-RTT HTTP client
+]
+
+[project.optional-dependencies]
+# EXP-02 code-metrics logistic-regression baseline.
+stats = [
+  "scikit-learn>=1.5",
+  "scipy>=1.14",
+]
+# M0.6 model acquisition and the S12-RTT ONNX encoder worker.
+# Not installed by default: these wheels are large and platform-specific,
+# and M0.2 is forbidden from vendoring any model weight.
+models = [
+  "huggingface-hub>=0.26",
+  "onnxruntime>=1.20",
+  "tokenizers>=0.20",
+  "transformers>=4.46",
+]
+dev = [
+  "pytest>=8.3",
+  "pytest-cov>=5.0",
+  "ruff>=0.7",
+]
+
+[tool.setuptools]
+package-dir = { "" = "src" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["anvil_eval*"]
+
+[tool.pytest.ini_options]
+minversion = "8.0"
+testpaths = ["tests"]
+addopts = "-ra --strict-markers --strict-config"
+markers = [
+  "network: test reaches the public internet; skipped in offline runs",
+  "docker: test shells out to Docker (ARVO reproducers, EXP-04)",
+  "model: test loads model weights from eval/models/",
+  "slow: test takes more than a few seconds",
+]
+
+[tool.ruff]
+line-length = 100
+src = ["src", "tests"]
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP"]