From 2daa17e2c7a64270f87d5374d1c2e3ee9c079c47 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 15 Aug 2026 14:12:55 +0000 Subject: [PATCH 1/2] feat(benchmark): scored scene-timing corpus on the real clock Add `docgen benchmark` and `scripts/benchmark-scenes.sh` so clock and compile changes are measured, not guessed. The harness executes compiled construct() with the real `_TimedScene` methods and stub Manim mobjects. A committed baseline fails the command when wait-skips, overshoots, or hold pulses regress. A control case keeps the historical issue #66 dump visible so the scorer cannot go blind. Co-authored-by: jmjava --- AGENTS.md | 5 +- README.md | 2 + pyproject.toml | 2 +- scripts/benchmark-scenes.sh | 12 + src/docgen/benchmark_data/README.md | 28 ++ src/docgen/benchmark_data/baseline.json | 117 +++++++ src/docgen/cli.py | 88 +++++ src/docgen/scene_asset_validate.py | 21 ++ src/docgen/scene_benchmark.py | 437 ++++++++++++++++++++++++ src/docgen/scene_clock_harness.py | 380 +++++++++++++++++++++ tests/test_scene_benchmark.py | 131 +++++++ 11 files changed, 1220 insertions(+), 3 deletions(-) create mode 100755 scripts/benchmark-scenes.sh create mode 100644 src/docgen/benchmark_data/README.md create mode 100644 src/docgen/benchmark_data/baseline.json create mode 100644 src/docgen/scene_benchmark.py create mode 100644 src/docgen/scene_clock_harness.py create mode 100644 tests/test_scene_benchmark.py diff --git a/AGENTS.md b/AGENTS.md index a924de8..8fc6ea0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,6 +58,7 @@ Commands registered on the **`docgen`** CLI include: - **`pages`** — emit static HTML for demo assets. - **`generate-all`** — orchestrated pipeline: TTS → timestamps → **scene specs** (auto `scene-spec-generate` when `animations/specs/` is empty; otherwise offline retime) → images → Manim → compose → validate → concat → pages. `--regen-scene-specs` forces LLM rewrite; `--skip-scene-retime` keeps legacy hand `scenes.py` only. - **`rebuild-after-audio`** — same as generate-all with TTS skipped (still retimes scenes after timestamps). +- **`benchmark`** — score the packaged scene-timing corpus (no bundle / Manim / OpenAI). Executes compiled ``construct()`` on the real ``_TimedScene`` clock and diffs ``src/docgen/benchmark_data/baseline.json``. Exit 1 on regression. ``--update-baseline`` only after an intentional improvement. ## Implications for changes here @@ -70,14 +71,14 @@ Commands registered on the **`docgen`** CLI include: ## Testing (downstream relevance) -Tests should cover **CLI-visible behavior** and contracts that adopters rely on: **`yaml-generate`**, **`scene-spec-generate`**, **`scene-compile`**, **`validate`**, **`compose`**, **`generate-all`**, **`pages`**, **`init`**, **config** loading (`repo_root`, `env_file`), and package exports. Use small in-tree fixtures; this library does not ship a dogfood bundle. +Tests should cover **CLI-visible behavior** and contracts that adopters rely on: **`yaml-generate`**, **`scene-spec-generate`**, **`scene-compile`**, **`validate`**, **`compose`**, **`generate-all`**, **`pages`**, **`init`**, **`benchmark`**, **config** loading (`repo_root`, `env_file`), and package exports. Use small in-tree fixtures; this library does not ship a dogfood bundle. Clock / compile changes must keep ``docgen benchmark`` at or above ``benchmark_data/baseline.json`` — string assertions on ``scenes.py`` are not enough. ## Cursor Cloud specific instructions - **Virtualenv:** the project is installed editable into **`/workspace/.venv`** (created by the startup update script). Shells do **not** auto-activate it — run `. /workspace/.venv/bin/activate` (or prefix the venv path) before `docgen`, `pytest`, or `ruff`. The `docgen` console script lives at `/workspace/.venv/bin/docgen`. - **System deps are pre-baked in the VM snapshot** (not the update script): `ffmpeg` + `tesseract-ocr` (validation/compose/OCR), plus `build-essential`, `python3-dev`, `libcairo2-dev`, `libpango1.0-dev`, `pkg-config` (needed to build the `manim` extra's `manimpango`/`pycairo` wheels). If a fresh VM ever lacks these, reinstall via apt before `pip install`. - **Standard commands** are in `README.md` / `pyproject.toml` / `.github/workflows/ci.yml`: lint `ruff check src/ tests/`; tests `pytest tests/ -v --tb=short`; the CI unit job also exports `PYTHONPATH=src` (not needed locally because of the editable install, but harmless). -- **OpenAI-gated vs offline commands:** `tts`, `timestamps --engine whisper`, `image-generate`, `narration-generate`, `scene-spec-generate`, and `yaml-generate --llm` call OpenAI and need `OPENAI_API_KEY` (integration tests auto-skip without it). Fully offline: `init`, `scene-compile`, `manim`, `compose`, `validate`, `lint`, `pages`, `concat`, `yaml-generate` (no `--llm`), and `timestamps` (default `local` engine). +- **OpenAI-gated vs offline commands:** `tts`, `timestamps --engine whisper`, `image-generate`, `narration-generate`, `scene-spec-generate`, and `yaml-generate --llm` call OpenAI and need `OPENAI_API_KEY` (integration tests auto-skip without it). Fully offline: `init`, `scene-compile`, `manim`, `compose`, `validate`, `lint`, `pages`, `concat`, `yaml-generate` (no `--llm`), `timestamps` (default `local` engine), and `benchmark`. - **`scene-compile` gotcha:** paced specs (`wait_word`) need a `timing.json` entry for that stem (`docgen timestamps` after TTS). Prefer `scene-compile --retime` after fresh timestamps; for a fully offline smoke render, author rows without wait indices only if you accept unpaced reveals. - **No in-repo dogfood bundle:** exercise the pipeline against a scratch bundle (`docgen init /tmp/ --defaults` in a throwaway git dir). Do not hand-edit consumer generated assets (see `.cursor/rules/no-asset-edits.mdc`). - **Wizard:** `docgen wizard --port 8501` is an optional local Flask app (long-lived); run it from a bundle directory that contains `docgen.yaml`. diff --git a/README.md b/README.md index c2bde75..b0e3890 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ python3 -m venv .venv source .venv/bin/activate pip install -e ".[dev]" pytest +docgen benchmark # standard scene-timing corpus (no Manim / OpenAI) ``` CI installs `ffmpeg` and `tesseract` via apt — see `.github/workflows/ci.yml`. @@ -140,6 +141,7 @@ docgen validate --pre-push | `docgen yaml-generate [--merge-defaults] [--llm] [--dry-run] [--list-gaps]` | Merge defaults into `docgen.yaml`; optional OpenAI refresh of `tts.instructions` / `wizard.system_prompt` (rewrites the file — review in Git) | | `docgen scene-compile [SPEC.scene.yaml \| --all] [--retime] [--dry-run]` | Compile declarative scene YAML into `animations/scenes.py`. **`--all --retime`** re-derives `wait_word` from current `timing.json` with no OpenAI; unmatched labels fail closed (or set `pace: none`) | | `docgen scene-spec-generate [--segment 01 \| --all] [--compile] [--print-only] [--output PATH] [--hint …] [--model …]` | Call OpenAI to emit YAML only (same schema as `scene-compile`); rejects frame-budget overflow and **subject-beat coverage** failures (hold board on same topic; cover topic shifts; no invented labels — not a blind count); auto-paginate + word-alignment; optionally writes `animations/specs/.scene.yaml` and `--compile`s into `scenes.py` | +| `docgen benchmark [--case ID] [--format text\|json] [--update-baseline]` | Score the **standard scene-timing corpus** (execute compiled `construct()` on the real `_TimedScene` clock, no Manim). Diffs a committed baseline so clock changes are measured, not guessed | ## Configuration diff --git a/pyproject.toml b/pyproject.toml index 304b59c..4267929 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ docgen = "docgen.cli:main" where = ["src"] [tool.setuptools.package-data] -docgen = ["templates/*.html", "static/*.css", "static/*.js"] +docgen = ["templates/*.html", "static/*.css", "static/*.js", "benchmark_data/*"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/scripts/benchmark-scenes.sh b/scripts/benchmark-scenes.sh new file mode 100755 index 0000000..400f085 --- /dev/null +++ b/scripts/benchmark-scenes.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Score the standard scene-timing corpus (no bundle, no Manim, no OpenAI). +# Usage: +# ./scripts/benchmark-scenes.sh +# ./scripts/benchmark-scenes.sh --format json +# ./scripts/benchmark-scenes.sh --update-baseline # after an intentional improvement +set -euo pipefail +root="$(cd "$(dirname "$0")/.." && pwd)" +if [[ -x "${root}/.venv/bin/docgen" ]]; then + exec "${root}/.venv/bin/docgen" benchmark "$@" +fi +exec docgen benchmark "$@" diff --git a/src/docgen/benchmark_data/README.md b/src/docgen/benchmark_data/README.md new file mode 100644 index 0000000..656e949 --- /dev/null +++ b/src/docgen/benchmark_data/README.md @@ -0,0 +1,28 @@ +# Scene-timing benchmark corpus + +Standard scripts scored by ``docgen benchmark`` / ``scripts/benchmark-scenes.sh``. + +Each case is a **fixed spec + Whisper-shaped word list** that encodes a +production failure we have already shipped by accident: + +| id | What it proves | +| --- | --- | +| `issue66_tight_clamped` | Tight spoken labels + long authored `run_time` must **not** skip waits after compile clamp | +| `issue66_tight_unclamped` | **Control** — same spec compiled *without* words still dumps (scorer is not blind) | +| `early_title` | Title `Write` must shrink so a 0.55s first label is not skipped | +| `wide_hold` | Long subject-beat holds get more than one pulse | +| `emphasis_none` | Author opt-out stays still (no false “stuck” fail) | +| `paged_slide` | Slide page transition does not skip the next `wait_word` | +| `flow_edges` | Grow + edge-to-edge arrow still paces | +| `audio_tail` | Last box keeps moving through a long tail | + +`baseline.json` is the last accepted scorecard. A change that raises +`defect_points` or drops `quality_points` fails the command (exit 1). + +After an **intentional** improvement: + +```bash +docgen benchmark --update-baseline +``` + +Review the baseline diff in Git. Do not bump it to hide a regression. diff --git a/src/docgen/benchmark_data/baseline.json b/src/docgen/benchmark_data/baseline.json new file mode 100644 index 0000000..51c21f1 --- /dev/null +++ b/src/docgen/benchmark_data/baseline.json @@ -0,0 +1,117 @@ +{ + "version": 1, + "cases": { + "issue66_tight_clamped": { + "role": "quality", + "wait_skips": 0, + "overshoots": 0, + "hold_idle_violations": 0, + "cadence_violations": 0, + "sim_drift": 0, + "mid_hold_pulses": 9, + "box_reveals": 3, + "last_motion_frac": 0.9877, + "defect_points": 0, + "quality_points": 20, + "score": 100 + }, + "issue66_tight_unclamped": { + "role": "control", + "wait_skips": 2, + "overshoots": 2, + "hold_idle_violations": 0, + "cadence_violations": 1, + "sim_drift": 1, + "mid_hold_pulses": 0, + "box_reveals": 3, + "last_motion_frac": 0.1407, + "defect_points": 80, + "quality_points": 0, + "score": 20 + }, + "early_title": { + "role": "quality", + "wait_skips": 0, + "overshoots": 0, + "hold_idle_violations": 0, + "cadence_violations": 0, + "sim_drift": 0, + "mid_hold_pulses": 4, + "box_reveals": 2, + "last_motion_frac": 0.9855, + "defect_points": 0, + "quality_points": 16, + "score": 100 + }, + "wide_hold": { + "role": "quality", + "wait_skips": 0, + "overshoots": 0, + "hold_idle_violations": 0, + "cadence_violations": 0, + "sim_drift": 0, + "mid_hold_pulses": 8, + "box_reveals": 3, + "last_motion_frac": 0.9951, + "defect_points": 0, + "quality_points": 20, + "score": 100 + }, + "emphasis_none": { + "role": "quality", + "wait_skips": 0, + "overshoots": 0, + "hold_idle_violations": 0, + "cadence_violations": 0, + "sim_drift": 0, + "mid_hold_pulses": 0, + "box_reveals": 2, + "last_motion_frac": 0.5399, + "defect_points": 0, + "quality_points": 4, + "score": 100 + }, + "paged_slide": { + "role": "quality", + "wait_skips": 0, + "overshoots": 0, + "hold_idle_violations": 0, + "cadence_violations": 0, + "sim_drift": 0, + "mid_hold_pulses": 4, + "box_reveals": 2, + "last_motion_frac": 0.9885, + "defect_points": 0, + "quality_points": 16, + "score": 100 + }, + "flow_edges": { + "role": "quality", + "wait_skips": 0, + "overshoots": 0, + "hold_idle_violations": 0, + "cadence_violations": 0, + "sim_drift": 0, + "mid_hold_pulses": 4, + "box_reveals": 2, + "last_motion_frac": 0.9903, + "defect_points": 0, + "quality_points": 16, + "score": 100 + }, + "audio_tail": { + "role": "quality", + "wait_skips": 0, + "overshoots": 0, + "hold_idle_violations": 0, + "cadence_violations": 0, + "sim_drift": 0, + "mid_hold_pulses": 5, + "box_reveals": 1, + "last_motion_frac": 0.9935, + "defect_points": 0, + "quality_points": 18, + "score": 100 + } + } +} diff --git a/src/docgen/cli.py b/src/docgen/cli.py index 2d11e8d..5cfcd50 100644 --- a/src/docgen/cli.py +++ b/src/docgen/cli.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os from pathlib import Path @@ -1140,3 +1141,90 @@ def rebuild_after_audio(ctx: click.Context, regen_scene_specs: bool) -> None: pipeline = Pipeline(cfg) pipeline.run(skip_tts=True, regen_scene_specs=regen_scene_specs) + +@main.command("benchmark") +@click.option( + "--case", + "case_id", + default=None, + help="Run one standard case id (default: the full corpus).", +) +@click.option( + "--format", + "fmt", + type=click.Choice(["text", "json"], case_sensitive=False), + default="text", +) +@click.option( + "--update-baseline", + is_flag=True, + help="Rewrite the committed baseline from this run (only after an intentional improvement).", +) +@click.option( + "--baseline", + "baseline_path", + default=None, + type=click.Path(dir_okay=False, path_type=Path), + help="Baseline JSON to compare (default: packaged benchmark_data/baseline.json).", +) +@click.option( + "--output", + "output_path", + default=None, + type=click.Path(dir_okay=False, path_type=Path), + help="Write the full JSON report to this path.", +) +def benchmark( + case_id: str | None, + fmt: str, + update_baseline: bool, + baseline_path: Path | None, + output_path: Path | None, +) -> None: + """Score the standard scene-timing corpus (no bundle, no Manim, no OpenAI). + + Executes compiled construct() on the real _TimedScene clock with stub + mobjects. Compare against the committed baseline to see if a change + improved or regressed production failure modes (issue #66 dumps, stuck + holds, title skip, page transitions). + """ + from docgen.scene_benchmark import ( + compare_to_baseline, + default_baseline_path, + format_table, + load_baseline, + run_benchmark, + scores_as_json, + write_baseline, + ) + + try: + scores = run_benchmark(case_id=case_id) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + base_path = baseline_path or default_baseline_path() + if update_baseline: + if case_id: + raise click.ClickException("--update-baseline requires the full corpus (omit --case)") + written = write_baseline(scores, base_path) + click.echo(f"wrote baseline {written}") + baseline = load_baseline(base_path) + regressions = compare_to_baseline(scores, baseline) + report = scores_as_json(scores, regressions=regressions) + if output_path: + output_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + if fmt == "json": + click.echo(json.dumps(report, indent=2)) + else: + click.echo(format_table(scores)) + if regressions: + click.echo("") + click.echo("regressions vs baseline:") + for note in regressions: + click.echo(f" - {note}") + elif base_path.is_file(): + click.echo("") + click.echo(f"meets baseline {base_path}") + if regressions and not update_baseline: + raise SystemExit(1) + diff --git a/src/docgen/scene_asset_validate.py b/src/docgen/scene_asset_validate.py index 7f6cc4c..b08ccc7 100644 --- a/src/docgen/scene_asset_validate.py +++ b/src/docgen/scene_asset_validate.py @@ -317,6 +317,27 @@ def scene_asset_violations_for_segment(cfg: "Config", seg_id: str) -> list[str]: ) issues.extend(dwell_overshoot_violations(events, audio_end=audio_end or None)) issues.extend(hold_idle_violations(events, audio_end=audio_end)) + merged_clock = dict(spec) + if not merged_clock.get("timing_key"): + merged_clock["timing_key"] = stem + try: + from docgen.scene_clock_harness import ( + clock_contract_violations, + run_compiled_scene_clock, + ) + from docgen.scene_spec import compile_scene_class + + compiled = compile_scene_class(merged_clock, words=words) + trace = run_compiled_scene_clock(compiled, words) + issues.extend( + f"clock: {m}" + for m in clock_contract_violations(trace, words, audio_end=audio_end) + ) + except Exception as exc: # noqa: BLE001 — fail closed before Manim + issues.append( + f"clock: compiled construct failed to execute ({exc}) — " + "run `docgen scene-compile --retime`" + ) if scenes_text: merged = dict(spec) if not merged.get("timing_key"): diff --git a/src/docgen/scene_benchmark.py b/src/docgen/scene_benchmark.py new file mode 100644 index 0000000..894eab0 --- /dev/null +++ b/src/docgen/scene_benchmark.py @@ -0,0 +1,437 @@ +"""Fixed scene-timing benchmark corpus and scorer. + +Run ``docgen benchmark`` after a clock / compile change. Cases are **standard +scripts** (spec + Whisper-shaped words) that encode production failure modes. +The scorer executes compiled ``construct()`` on the real ``_TimedScene`` clock +(see ``scene_clock_harness``) and diffs the result against a committed baseline. + +Quality cases must not grow defects or lose quality points. Control cases must +keep failing — they prove the scorer still detects the historical bug. +""" + +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +from docgen.manim_primitives import audio_end_from_words +from docgen.scene_asset_validate import hold_idle_violations +from docgen.scene_clock_harness import ( + ClockTrace, + clock_contract_violations, + run_compiled_scene_clock, + simulator_exec_drift_violations, +) +from docgen.scene_spec import ( + compile_scene_class, + reveal_cadence_violations, + simulate_reveal_timeline, +) + +BASELINE_NAME = "baseline.json" + + +def benchmark_data_dir() -> Path: + return Path(__file__).resolve().parent / "benchmark_data" + + +def default_baseline_path() -> Path: + return benchmark_data_dir() / BASELINE_NAME + + +@dataclass(frozen=True) +class BenchmarkCase: + """One standard script: declarative spec + timing words.""" + + id: str + title: str + spec: dict[str, Any] + words: list[dict[str, Any]] + role: str = "quality" # quality | control + compile_with_words: bool = True + + +@dataclass +class CaseScore: + case_id: str + title: str + role: str + wait_skips: int + overshoots: int + hold_idle_violations: int + cadence_violations: int + sim_drift: int + mid_hold_pulses: int + box_reveals: int + last_motion_frac: float + audio_end: float + defect_points: int + quality_points: int + score: int + issues: list[str] = field(default_factory=list) + + def snapshot(self) -> dict[str, Any]: + return { + "role": self.role, + "wait_skips": self.wait_skips, + "overshoots": self.overshoots, + "hold_idle_violations": self.hold_idle_violations, + "cadence_violations": self.cadence_violations, + "sim_drift": self.sim_drift, + "mid_hold_pulses": self.mid_hold_pulses, + "box_reveals": self.box_reveals, + "last_motion_frac": round(self.last_motion_frac, 4), + "defect_points": self.defect_points, + "quality_points": self.quality_points, + "score": self.score, + } + + +def _box( + label: str, + *, + wait_word: int | None = None, + emphasis: str | None = None, + reveal: str | None = None, + shape: str | None = None, + color: str = "C_GREEN", +) -> dict[str, Any]: + out: dict[str, Any] = { + "label": label, + "color": color, + "width": 3.0, + "height": 0.8, + "font_size": 18, + } + if wait_word is not None: + out["wait_word"] = wait_word + if emphasis is not None: + out["emphasis"] = emphasis + if reveal is not None: + out["reveal"] = reveal + if shape is not None: + out["shape"] = shape + return out + + +def _spec( + case_id: str, + boxes: list[dict[str, Any]], + *, + layout: dict[str, Any] | None = None, + pages: list[dict[str, Any]] | None = None, + edges: list[dict[str, Any]] | None = None, + run_time: float = 1.5, +) -> dict[str, Any]: + spec: dict[str, Any] = { + "segment_id": "01", + "class_name": "BenchScene", + "timing_key": case_id, + "title": {"text": "Benchmark", "font_size": 36, "color": "C_WHITE"}, + } + if pages is not None: + spec["pages"] = pages + else: + spec["rows"] = [{"run_time": run_time, "boxes": boxes}] + if layout: + spec["layout"] = layout + if edges: + spec["edges"] = edges + return spec + + +def standard_cases() -> list[BenchmarkCase]: + """Committed corpus. Add a case here when a new production failure shows up.""" + tight = [ + {"word": "Alpha", "start": 1.2, "end": 1.4}, + {"word": "Beta", "start": 1.6, "end": 1.8}, + {"word": "Gamma", "start": 2.0, "end": 2.2}, + {"word": "tail", "start": 40.0, "end": 40.5}, + ] + cascade_boxes = [ + _box("Alpha", wait_word=0), + _box("Beta", wait_word=1), + _box("Gamma", wait_word=2), + ] + wide = [ + {"word": "Alpha", "start": 1.2, "end": 1.4}, + {"word": "Beta", "start": 8.0, "end": 8.3}, + {"word": "Gamma", "start": 16.0, "end": 16.3}, + {"word": "tail", "start": 24.0, "end": 24.4}, + ] + return [ + BenchmarkCase( + id="issue66_tight_clamped", + title="Tight words + compile clamp (issue #66 must stay fixed)", + spec=_spec("issue66_tight_clamped", cascade_boxes, run_time=1.5), + words=tight, + role="quality", + compile_with_words=True, + ), + BenchmarkCase( + id="issue66_tight_unclamped", + title="Control: same spec compiled without words (historical dump)", + spec=_spec("issue66_tight_unclamped", cascade_boxes, run_time=1.5), + words=tight, + role="control", + compile_with_words=False, + ), + BenchmarkCase( + id="early_title", + title="First spoken label at 0.55s — title Write must not skip it", + spec=_spec( + "early_title", + [_box("Alpha", wait_word=0), _box("Beta", wait_word=1)], + run_time=0.8, + ), + words=[ + {"word": "Alpha", "start": 0.55, "end": 0.7}, + {"word": "Beta", "start": 4.0, "end": 4.2}, + {"word": "tail", "start": 8.0, "end": 8.3}, + ], + ), + BenchmarkCase( + id="wide_hold", + title="Long subject-beat holds must pulse more than once", + spec=_spec("wide_hold", cascade_boxes, run_time=0.8), + words=wide, + ), + BenchmarkCase( + id="emphasis_none", + title="Author opt-out: wide holds with emphasis none stay still", + spec=_spec( + "emphasis_none", + [ + _box("Alpha", wait_word=0, emphasis="none"), + _box("Beta", wait_word=1, emphasis="none"), + ], + layout={"dwell_emphasis": "none"}, + run_time=0.8, + ), + words=wide[:3], + ), + BenchmarkCase( + id="paged_slide", + title="Two pages with slide transition; second wait must still fire", + spec=_spec( + "paged_slide", + [], + layout={"page_transition": "slide", "page_transition_run_time": 0.4}, + pages=[ + { + "rows": [ + {"run_time": 0.6, "boxes": [_box("Alpha", wait_word=0)]} + ] + }, + { + "transition": "slide", + "rows": [ + {"run_time": 0.6, "boxes": [_box("Beta", wait_word=1)]} + ], + }, + ], + ), + words=[ + {"word": "Alpha", "start": 1.2, "end": 1.4}, + {"word": "Beta", "start": 6.0, "end": 6.3}, + {"word": "tail", "start": 10.0, "end": 10.4}, + ], + ), + BenchmarkCase( + id="flow_edges", + title="Pipeline: first node grows, edge-to-edge arrow, paced holds", + spec=_spec( + "flow_edges", + [ + _box("Hints", wait_word=0, reveal="grow"), + _box("YAML", wait_word=1, color="C_BLUE"), + ], + edges=[{"from": "Hints", "to": "YAML", "color": "C_ACCENT"}], + run_time=0.7, + ), + words=[ + {"word": "Hints", "start": 1.5, "end": 1.8}, + {"word": "YAML", "start": 7.0, "end": 7.3}, + {"word": "tail", "start": 12.0, "end": 12.4}, + ], + ), + BenchmarkCase( + id="audio_tail", + title="Last box holds through a long audio tail", + spec=_spec("audio_tail", [_box("Alpha", wait_word=0)], run_time=0.6), + words=[ + {"word": "Alpha", "start": 1.2, "end": 1.5}, + {"word": "tail", "start": 18.0, "end": 18.5}, + ], + ), + ] + + +def score_case(case: BenchmarkCase) -> CaseScore: + words = case.words + compile_words = words if case.compile_with_words else None + src = compile_scene_class(case.spec, words=compile_words) + trace: ClockTrace = run_compiled_scene_clock(src, words) + audio_end = float(audio_end_from_words(words) or 0.0) + issues = clock_contract_violations(trace, words, audio_end=audio_end) + sim = simulate_reveal_timeline(case.spec, words, clamp_run_times=case.compile_with_words) + drift = simulator_exec_drift_violations(sim, trace) + cadence = reveal_cadence_violations(sim, audio_end=audio_end) + idle = hold_idle_violations(sim, audio_end=audio_end) + overshoots = sum(1 for msg in issues if msg.startswith("overshoot:")) + last_motion = 0.0 + for ev in trace.play_events(): + if not ev.anim_kinds or "FadeOut" in ev.anim_kinds: + continue + last_motion = max(last_motion, ev.clock_after) + frac = (last_motion / audio_end) if audio_end > 0 else 0.0 + wait_skips = len(trace.skipped_waits()) + defect_points = ( + 15 * wait_skips + + 15 * overshoots + + 10 * len(idle) + + 10 * len(cadence) + + 10 * len(drift) + ) + quality_points = 0 + if case.role == "quality": + quality_points = min(12, len(trace.emphasis_plays()) * 2) + min( + 8, int(round(min(frac, 1.0) * 8)) + ) + score = max(0, min(100, 100 - defect_points + quality_points)) + return CaseScore( + case_id=case.id, + title=case.title, + role=case.role, + wait_skips=wait_skips, + overshoots=overshoots, + hold_idle_violations=len(idle), + cadence_violations=len(cadence), + sim_drift=len(drift), + mid_hold_pulses=len(trace.emphasis_plays()), + box_reveals=len(trace.reveal_plays()), + last_motion_frac=frac, + audio_end=audio_end, + defect_points=defect_points, + quality_points=quality_points, + score=score, + issues=issues + [f"drift: {m}" for m in drift] + cadence + idle, + ) + + +def run_benchmark( + *, + case_id: str | None = None, +) -> list[CaseScore]: + cases = standard_cases() + if case_id: + cases = [c for c in cases if c.id == case_id] + if not cases: + known = ", ".join(c.id for c in standard_cases()) + raise ValueError(f"unknown benchmark case {case_id!r}; known: {known}") + return [score_case(c) for c in cases] + + +def load_baseline(path: Path | None = None) -> dict[str, Any]: + p = path or default_baseline_path() + if not p.is_file(): + return {"version": 1, "cases": {}} + return json.loads(p.read_text(encoding="utf-8")) + + +def dump_baseline(scores: list[CaseScore]) -> dict[str, Any]: + return { + "version": 1, + "cases": {s.case_id: s.snapshot() for s in scores}, + } + + +def write_baseline(scores: list[CaseScore], path: Path | None = None) -> Path: + p = path or default_baseline_path() + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps(dump_baseline(scores), indent=2) + "\n", encoding="utf-8") + return p + + +def compare_to_baseline( + scores: list[CaseScore], + baseline: dict[str, Any], +) -> list[str]: + """Return regression notes. Empty means the run meets or beats the baseline.""" + notes: list[str] = [] + stored = baseline.get("cases") if isinstance(baseline.get("cases"), dict) else {} + for score in scores: + prev = stored.get(score.case_id) + if not isinstance(prev, dict): + notes.append(f"{score.case_id}: missing from baseline (run --update-baseline)") + continue + if score.role == "control": + prev_defects = int(prev.get("defect_points", 0)) + if score.defect_points <= 0: + notes.append( + f"{score.case_id}: control no longer fails — scorer may be blind " + "to the historical dump" + ) + elif score.defect_points < prev_defects // 2 and prev_defects >= 20: + notes.append( + f"{score.case_id}: control defects dropped sharply " + f"({prev_defects} → {score.defect_points}); confirm this is intended" + ) + continue + if score.defect_points > int(prev.get("defect_points", 0)): + notes.append( + f"{score.case_id}: defect_points {prev.get('defect_points')} → " + f"{score.defect_points} (regression)" + ) + if score.quality_points < int(prev.get("quality_points", 0)): + notes.append( + f"{score.case_id}: quality_points {prev.get('quality_points')} → " + f"{score.quality_points} (worse motion / tail coverage)" + ) + if score.score < int(prev.get("score", 0)): + notes.append( + f"{score.case_id}: score {prev.get('score')} → {score.score}" + ) + if score.wait_skips > int(prev.get("wait_skips", 0)): + notes.append( + f"{score.case_id}: wait_skips {prev.get('wait_skips')} → {score.wait_skips}" + ) + if score.mid_hold_pulses < int(prev.get("mid_hold_pulses", 0)): + notes.append( + f"{score.case_id}: mid_hold_pulses {prev.get('mid_hold_pulses')} → " + f"{score.mid_hold_pulses} (holds look more stuck)" + ) + return notes + + +def format_table(scores: list[CaseScore]) -> str: + headers = ( + f"{'case':<26} {'role':<8} {'skip':>4} {'ovr':>4} {'idle':>4} " + f"{'drift':>5} {'pulse':>5} {'q':>3} {'def':>4} {'score':>5}" + ) + lines = [headers, "-" * len(headers)] + for s in scores: + lines.append( + f"{s.case_id:<26} {s.role:<8} {s.wait_skips:>4} {s.overshoots:>4} " + f"{s.hold_idle_violations:>4} {s.sim_drift:>5} {s.mid_hold_pulses:>5} " + f"{s.quality_points:>3} {s.defect_points:>4} {s.score:>5}" + ) + quality = [s for s in scores if s.role == "quality"] + if quality: + avg = sum(s.score for s in quality) / len(quality) + lines.append("-" * len(headers)) + lines.append(f"{'quality average':<26} {'':<8} {'':>4} {'':>4} {'':>4} {'':>5} {'':>5} {'':>3} {'':>4} {avg:5.1f}") + return "\n".join(lines) + + +def scores_as_json(scores: list[CaseScore], *, regressions: list[str]) -> dict[str, Any]: + return { + "cases": [asdict(s) for s in scores], + "quality_average": ( + sum(s.score for s in scores if s.role == "quality") + / max(1, sum(1 for s in scores if s.role == "quality")) + ), + "regressions": regressions, + } diff --git a/src/docgen/scene_clock_harness.py b/src/docgen/scene_clock_harness.py new file mode 100644 index 0000000..4a779bf --- /dev/null +++ b/src/docgen/scene_clock_harness.py @@ -0,0 +1,380 @@ +"""Execute compiled scene ``construct()`` against the real ``_TimedScene`` clock. + +Unit tests that only inspect source strings or ``simulate_reveal_timeline`` +have passed while consumer renders still dumped the first board or froze. +This harness runs the **compiled class** with stub Manim mobjects and the +**actual** ``_TimedScene`` methods from ``BOOTSTRAP_HEADER`` — no Manim +install, no ffmpeg. The clock that production uses is the clock we score. +""" + +from __future__ import annotations + +import ast +from dataclasses import dataclass, field +from types import SimpleNamespace +from typing import Any + + +@dataclass(frozen=True) +class ClockEvent: + """One ``play`` / ``wait`` / ``wait_word`` observed during ``construct``.""" + + kind: str + clock_before: float + duration: float + anim_kinds: tuple[str, ...] = () + skipped: bool = False + word_index: int | None = None + word_start: float | None = None + + @property + def clock_after(self) -> float: + return float(self.clock_before) + float(self.duration) + + +@dataclass +class ClockTrace: + """Full construct run: events plus the final ``_TimedScene._clock``.""" + + events: list[ClockEvent] = field(default_factory=list) + final_clock: float = 0.0 + + def wait_word_events(self) -> list[ClockEvent]: + return [e for e in self.events if e.kind == "wait_word"] + + def play_events(self) -> list[ClockEvent]: + return [e for e in self.events if e.kind == "play"] + + def skipped_waits(self) -> list[ClockEvent]: + return [e for e in self.wait_word_events() if e.skipped] + + def emphasis_plays(self) -> list[ClockEvent]: + return [ + e + for e in self.play_events() + if any(k in {"Indicate", "Circumscribe"} for k in e.anim_kinds) + ] + + def reveal_plays(self) -> list[ClockEvent]: + reveal_kinds = {"FadeIn", "GrowFromCenter"} + return [ + e + for e in self.play_events() + if any(k in reveal_kinds for k in e.anim_kinds) + and "FadeOut" not in e.anim_kinds + ] + + +class _Dummy: + """Swallow Manim layout / animation calls; carry a kind tag for scoring.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + self._kind = str(kwargs.pop("_kind", "mobject")) + self._role = kwargs.pop("_role", None) + self._target = args[0] if args else None + self._label = kwargs.get("label") + + def __getattr__(self, name: str) -> Any: + def _method(*_a: Any, **_k: Any) -> _Dummy: + return self + + return _method + + def __mul__(self, other: Any) -> _Dummy: + return self + + def __rmul__(self, other: Any) -> _Dummy: + return self + + def __neg__(self) -> _Dummy: + return self + + def __iter__(self) -> Any: + return iter(()) + + +class StubScene: + """Stand-in for ``manim.Scene`` so ``_TimedScene`` can run offline.""" + + def __init__(self) -> None: + self._clock = 0.0 + self.mobjects: list[Any] = [] + self.camera = SimpleNamespace(background_color=None) + self.events: list[ClockEvent] = [] + self.setup() + + def setup(self) -> None: + self._clock = 0.0 + + def play(self, *animations: Any, run_time: float = 1.0, **_kwargs: Any) -> None: + kinds = tuple(getattr(a, "_kind", type(a).__name__) for a in animations) + self.events.append( + ClockEvent( + kind="play", + clock_before=float(self._clock), + duration=float(run_time), + anim_kinds=kinds, + ) + ) + + def wait(self, duration: float) -> None: + self.events.append( + ClockEvent( + kind="wait", + clock_before=float(self._clock), + duration=float(duration), + ) + ) + + def add(self, *mobjects: Any) -> None: + self.mobjects.extend(mobjects) + + def remove(self, *_mobjects: Any) -> None: + return None + + +def _anim(name: str) -> Any: + def factory(*args: Any, **kwargs: Any) -> _Dummy: + d = _Dummy(*args, **kwargs) + d._kind = name + if args: + d._target = args[0] + return d + + factory.__name__ = name + return factory + + +def _extract_timed_scene_src() -> str: + from docgen.manim_scene_support import BOOTSTRAP_HEADER + + tree = ast.parse(BOOTSTRAP_HEADER) + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name == "_TimedScene": + src = ast.get_source_segment(BOOTSTRAP_HEADER, node) + if not src: + break + return src.replace("class _TimedScene(Scene):", "class _TimedScene(StubScene):", 1) + raise RuntimeError("BOOTSTRAP_HEADER is missing class _TimedScene(Scene)") + + +def _exec_namespace( + words: list[dict[str, Any]], + segments: list[dict[str, Any]] | None, +) -> dict[str, Any]: + vec = _Dummy(_kind="dir") + ns: dict[str, Any] = { + "StubScene": StubScene, + "SimpleNamespace": SimpleNamespace, + "Text": _anim("Text"), + "Write": _anim("Write"), + "FadeIn": _anim("FadeIn"), + "FadeOut": _anim("FadeOut"), + "GrowFromCenter": _anim("GrowFromCenter"), + "GrowArrow": _anim("GrowArrow"), + "Indicate": _anim("Indicate"), + "Circumscribe": _anim("Circumscribe"), + "VGroup": _anim("VGroup"), + "Group": _anim("Group"), + "Arrow": _anim("Arrow"), + "DashedVMobject": _anim("DashedVMobject"), + "ImageMobject": _anim("ImageMobject"), + "UP": vec, + "DOWN": vec, + "LEFT": vec, + "RIGHT": vec, + "MANIM_FONT": "Liberation Sans", + "C_BG": "#1e1e2e", + "C_ACCENT": "#667eea", + "C_GREEN": "#42b883", + "C_ORANGE": "#f9a825", + "C_BLUE": "#2979ff", + "C_RED": "#ff5252", + "C_TEAL": "#26c6da", + "C_PURPLE": "#ce93d8", + "C_WHITE": "#cdd6f4", + "_box": lambda *a, **k: _Dummy(*a, _kind="box", _role="box", label=a[0] if a else ""), + "_arrow": lambda *a, **k: _Dummy(*a, _kind="arrow", _role="arrow"), + "_image": lambda *a, **k: _Dummy(*a, _kind="image", _role="image"), + "_load_timing_words": lambda _key: list(words), + "_load_timing": lambda _key: list(segments or []), + } + exec(_extract_timed_scene_src(), ns) # noqa: S102 — our bootstrap, not user input + timed = ns["_TimedScene"] + orig = timed.wait_until_word + + def _logged_wait_until_word(self: Any, words_arg: Any, index: int) -> None: + before = float(getattr(self, "_clock", 0.0)) + target: float | None = None + try: + if words_arg and 0 <= int(index) < len(words_arg): + target = float(words_arg[int(index)].get("start", 0.0)) + except (TypeError, ValueError, AttributeError): + target = None + orig(self, words_arg, index) + skipped = target is not None and before > target + 0.05 + self.events.append( + ClockEvent( + kind="wait_word", + clock_before=before, + duration=0.0, + skipped=skipped, + word_index=int(index), + word_start=target, + ) + ) + + timed.wait_until_word = _logged_wait_until_word + return ns + + +def run_compiled_scene_clock( + class_src: str, + words: list[dict[str, Any]], + *, + segments: list[dict[str, Any]] | None = None, +) -> ClockTrace: + """Exec ``class_src`` and run ``construct()``. Returns the observed clock.""" + ns = _exec_namespace(words, segments) + exec(class_src, ns) # noqa: S102 — compiler output, not user input + timed = ns["_TimedScene"] + cls = None + for value in ns.values(): + if isinstance(value, type) and value is not timed and issubclass(value, timed): + cls = value + break + if cls is None: + raise RuntimeError("compiled source did not define a _TimedScene subclass") + scene = cls() + scene.construct() + return ClockTrace(events=list(scene.events), final_clock=float(scene._clock)) + + +def clock_contract_violations( + trace: ClockTrace, + words: list[dict[str, Any]], + *, + audio_end: float = 0.0, + slack: float = 0.05, +) -> list[str]: + """Fail-closed checks on an **executed** construct (not the simulator).""" + from docgen.manim_primitives import ( + DWELL_CLOCK_MARGIN, + MAX_STATIC_HOLD, + MIN_DWELL_RUN_TIME, + audio_end_from_words, + ) + + issues: list[str] = [] + for ev in trace.skipped_waits(): + issues.append( + f"wait_until_word[{ev.word_index}] no-op: clock {ev.clock_before:.2f}s " + f"already past spoken start {ev.word_start:.2f}s (first-board dump / issue #66)" + ) + + waits = trace.wait_word_events() + for i, ev in enumerate(waits): + nxt = None + for later in waits[i + 1 :]: + if later.word_start is not None: + nxt = float(later.word_start) + break + if nxt is None: + continue + # Plays after this wait and before the next wait_word must not pass nxt. + started = False + for item in trace.events: + if item is ev: + started = True + continue + if not started: + continue + if item.kind == "wait_word": + break + if item.kind == "play" and item.clock_after > nxt + slack: + issues.append( + f"overshoot: {item.anim_kinds} ended at {item.clock_after:.2f}s " + f"past next wait_word start {nxt:.2f}s" + ) + + end = float(audio_end) if audio_end else float(audio_end_from_words(words) or 0.0) + allowed_idle = MAX_STATIC_HOLD + MIN_DWELL_RUN_TIME + DWELL_CLOCK_MARGIN + 0.15 + for i, ev in enumerate(waits): + nxt = None + for later in waits[i + 1 :]: + if later.word_start is not None: + nxt = float(later.word_start) + break + if nxt is None: + nxt = end if end > 0 else None + if nxt is None: + continue + window: list[ClockEvent] = [] + started = False + for item in trace.events: + if item is ev: + started = True + continue + if not started: + continue + if item.kind == "wait_word": + break + window.append(item) + last_motion = ev.clock_before + pulses = 0 + for item in window: + if item.kind == "play": + last_motion = item.clock_after + if any(k in {"Indicate", "Circumscribe"} for k in item.anim_kinds): + pulses += 1 + idle = float(nxt) - last_motion + # Author may set emphasis: none. Only fail when we *did* pulse, then froze. + if pulses >= 1 and idle > allowed_idle: + issues.append( + f"stuck hold: {idle:.2f}s idle after last pulse " + f"(wait_word[{ev.word_index}], max {allowed_idle:.2f}s)" + ) + return issues + + +def simulator_exec_drift_violations( + sim_events: list[Any], + trace: ClockTrace, + *, + slack: float = 0.12, +) -> list[str]: + """Fail when ``simulate_reveal_timeline`` disagrees with executed construct.""" + issues: list[str] = [] + sim_skips = sum(1 for e in sim_events if getattr(e, "wait_skipped", False)) + exec_skips = len(trace.skipped_waits()) + if sim_skips != exec_skips: + issues.append( + f"drift: simulator wait skips={sim_skips} executed skips={exec_skips}" + ) + sim_pulses = 0 + for ev in sim_events: + pulses = getattr(ev, "hold_pulses", ()) or () + if pulses: + sim_pulses += len(pulses) + elif float(getattr(ev, "dwell_run_time", 0.0) or 0.0) > 0: + sim_pulses += 1 + exec_pulses = len(trace.emphasis_plays()) + if sim_pulses != exec_pulses: + issues.append( + f"drift: simulator hold pulses={sim_pulses} executed Indicate/Circumscribe={exec_pulses}" + ) + paced_sim = [e for e in sim_events if getattr(e, "wait_word", None) is not None] + paced_exec = trace.wait_word_events() + for sim, exe in zip(paced_sim, paced_exec): + if getattr(sim, "wait_skipped", False) or exe.skipped: + continue + start = getattr(sim, "word_start", None) + if start is None or exe.word_start is None: + continue + # After a successful wait, executed clock should be on the spoken start. + if abs(float(exe.word_start) - float(start)) > slack: + issues.append( + f"drift: wait_word[{exe.word_index}] sim start {start:.2f} " + f"exec start {exe.word_start:.2f}" + ) + return issues diff --git a/tests/test_scene_benchmark.py b/tests/test_scene_benchmark.py new file mode 100644 index 0000000..74f36ba --- /dev/null +++ b/tests/test_scene_benchmark.py @@ -0,0 +1,131 @@ +"""Standard scene-timing benchmark: executed clock, not source-string checks.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from click.testing import CliRunner + +from docgen.cli import main +from docgen.scene_benchmark import ( + compare_to_baseline, + default_baseline_path, + format_table, + load_baseline, + run_benchmark, + score_case, + standard_cases, +) +from docgen.scene_clock_harness import run_compiled_scene_clock +from docgen.scene_spec import compile_scene_class + + +def test_corpus_ids_are_stable() -> None: + ids = [c.id for c in standard_cases()] + assert ids == [ + "issue66_tight_clamped", + "issue66_tight_unclamped", + "early_title", + "wide_hold", + "emphasis_none", + "paged_slide", + "flow_edges", + "audio_tail", + ] + + +def test_control_case_still_detects_issue66_dump() -> None: + case = next(c for c in standard_cases() if c.id == "issue66_tight_unclamped") + score = score_case(case) + assert score.role == "control" + assert score.wait_skips >= 2 + assert score.defect_points > 0 + assert any("no-op" in i for i in score.issues) + + +def test_clamped_issue66_case_does_not_skip_waits() -> None: + case = next(c for c in standard_cases() if c.id == "issue66_tight_clamped") + score = score_case(case) + assert score.wait_skips == 0 + assert score.overshoots == 0 + assert score.defect_points == 0 + + +def test_harness_runs_real_timed_scene_clock() -> None: + case = next(c for c in standard_cases() if c.id == "early_title") + src = compile_scene_class(case.spec, words=case.words) + trace = run_compiled_scene_clock(src, case.words) + waits = trace.wait_word_events() + assert waits + assert not any(w.skipped for w in waits) + # First spoken start is 0.55s; an unclamped 1.0s title Write would skip it. + assert waits[0].word_start == 0.55 + assert waits[0].clock_before <= 0.55 + 0.02 + + +def test_wide_hold_executes_more_than_one_pulse() -> None: + case = next(c for c in standard_cases() if c.id == "wide_hold") + score = score_case(case) + assert score.mid_hold_pulses >= 2 + assert score.wait_skips == 0 + + +def test_emphasis_none_emits_no_pulses() -> None: + case = next(c for c in standard_cases() if c.id == "emphasis_none") + score = score_case(case) + assert score.mid_hold_pulses == 0 + assert score.wait_skips == 0 + + +def test_full_corpus_meets_committed_baseline() -> None: + scores = run_benchmark() + baseline = load_baseline() + assert baseline.get("cases"), "baseline.json is missing — run docgen benchmark --update-baseline" + notes = compare_to_baseline(scores, baseline) + assert notes == [], "\n".join(notes) + table = format_table(scores) + assert "issue66_tight_clamped" in table + assert "quality average" in table + + +def test_compare_flags_skip_regression() -> None: + scores = run_benchmark() + dump = load_baseline() + cases = dict(dump["cases"]) + cases["issue66_tight_clamped"] = { + **cases["issue66_tight_clamped"], + "wait_skips": 0, + "defect_points": 0, + "score": 100, + } + # Pretend baseline was perfect; inject a worse score object. + worse = next(s for s in scores if s.case_id == "issue66_tight_clamped") + worse.wait_skips = 2 + worse.defect_points = 30 + worse.score = 70 + notes = compare_to_baseline([worse], {"version": 1, "cases": cases}) + assert notes + assert any("wait_skips" in n or "defect_points" in n or "score" in n for n in notes) + + +def test_cli_benchmark_text_and_json(tmp_path: Path) -> None: + runner = CliRunner() + result = runner.invoke(main, ["benchmark", "--format", "text"]) + assert result.exit_code == 0, result.output + assert "issue66_tight_clamped" in result.output + out = tmp_path / "report.json" + result = runner.invoke( + main, + ["benchmark", "--format", "json", "--output", str(out), "--case", "early_title"], + ) + assert result.exit_code == 0, result.output + payload = json.loads(out.read_text(encoding="utf-8")) + assert payload["cases"][0]["case_id"] == "early_title" + + +def test_packaged_baseline_exists() -> None: + path = default_baseline_path() + assert path.is_file() + data = json.loads(path.read_text(encoding="utf-8")) + assert set(data["cases"]) == {c.id for c in standard_cases()} From e98db5a6bf4a320f7cde626fb283716d8fb39d62 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 15 Aug 2026 15:34:59 +0000 Subject: [PATCH 2/2] chore(ci): require docgen benchmark on every change Add a dedicated CI job that runs `docgen benchmark`, lock that job in tests so it cannot be deleted quietly, and make the command a required gate in AGENTS.md and Cursor rules. Co-authored-by: jmjava --- .cursor/rules/docgen-benchmark.mdc | 27 +++++++++++++++++++++++++++ .cursor/rules/docgen-tools-only.mdc | 5 ++++- .github/workflows/ci.yml | 13 +++++++++++++ AGENTS.md | 15 ++++++++++++++- README.md | 5 +++-- src/docgen/benchmark_data/README.md | 5 +++++ tests/test_scene_benchmark.py | 22 ++++++++++++++++++++++ 7 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 .cursor/rules/docgen-benchmark.mdc diff --git a/.cursor/rules/docgen-benchmark.mdc b/.cursor/rules/docgen-benchmark.mdc new file mode 100644 index 0000000..8bddbd3 --- /dev/null +++ b/.cursor/rules/docgen-benchmark.mdc @@ -0,0 +1,27 @@ +--- +description: Clock and scene-compile changes must pass docgen benchmark against the committed baseline. +alwaysApply: true +--- + +# Required gate: `docgen benchmark` + +String checks on compiled `scenes.py` and `simulate_reveal_timeline` are **not** +enough. Production uses `_TimedScene._clock` inside `construct()`. + +**After any change** to scene compile, clock math, `_TimedScene` / bootstrap +helpers, dwell / hold pulses, `scene_asset_validate`, or the benchmark corpus +itself, you **must**: + +1. Run **`docgen benchmark`** (or **`./scripts/benchmark-scenes.sh`**). +2. Leave it green vs **`src/docgen/benchmark_data/baseline.json`** (exit 0). +3. Keep the **`benchmark`** job in **`.github/workflows/ci.yml`** (it must + invoke `docgen benchmark`). Do not delete or skip that job. +4. Keep **`tests/test_scene_benchmark.py`** in the default pytest run. + +`--update-baseline` is allowed **only** when the scorecard change is the point +of the PR. Review the baseline JSON diff. Do not bump the baseline to hide a +regression. + +When a consumer render fails in a new way, **add a case** to +`standard_cases()` in `src/docgen/scene_benchmark.py` and update the baseline +in the same PR. diff --git a/.cursor/rules/docgen-tools-only.mdc b/.cursor/rules/docgen-tools-only.mdc index 326cc1d..986966c 100644 --- a/.cursor/rules/docgen-tools-only.mdc +++ b/.cursor/rules/docgen-tools-only.mdc @@ -23,7 +23,10 @@ how a *consumer* bundle (typically `docs/demos/`) is expected to be operated. **Manim diagram segments:** use **`docgen scene-spec-generate`** / **`scene-compile`** with committed **`animations/specs/*.scene.yaml`** instead of hand-editing **`scenes.py`** inside generated regions. Subject-beat coverage (hold board on same topic; cover topic shifts; no invented labels) is enforced by **`scene-spec-generate`** and **`validate`** — not a blind label count. -**Surviving CLI commands (indicative):** `init`, `wizard`, `tts`, `timestamps`, `manim`, `compose`, `validate`, `lint`, `narration-generate`, `scene-spec-generate`, `scene-compile`, `yaml-generate`, `clean-bundle`, `concat`, `pages`, `generate-all`, `rebuild-after-audio`. +**Surviving CLI commands (indicative):** `init`, `wizard`, `tts`, `timestamps`, `manim`, `compose`, `validate`, `lint`, `narration-generate`, `scene-spec-generate`, `scene-compile`, `yaml-generate`, `clean-bundle`, `concat`, `pages`, `generate-all`, `rebuild-after-audio`, `benchmark`. + +**Clock / compile changes:** run **`docgen benchmark`** and keep the CI +`benchmark` job. See **`.cursor/rules/docgen-benchmark.mdc`**. **If the right command does not exist:** implement it in **docgen** (or a thin wrapper script committed to the consumer repo), then use it — do not bypass with manual file edits. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddec1ae..2565e75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,3 +60,16 @@ jobs: done - run: pip install --no-cache-dir ".[dev]" - run: pytest tests/ -v --tb=short + + # Required gate for clock / scene-compile changes. Do not remove: tests + # assert this job runs `docgen benchmark` against the committed baseline. + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - run: pip install --no-cache-dir "." + - name: Scene-timing benchmark + run: docgen benchmark diff --git a/AGENTS.md b/AGENTS.md index 8fc6ea0..e0b4fc0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,10 +65,23 @@ Commands registered on the **`docgen`** CLI include: - **Manim / `scenes.py` (marker blocks):** Fix generators under `src/docgen/**` (`manim_scene_support.py`, `scene_spec.py`, `scene_spec_generate.py`, `validate`, `yaml_generate`, tests). **Do not** patch generated classes inside a consumer's **`animations/scenes.py`** between **`BEGIN/END GENERATED SCENE`** markers; re-run **`scene-spec-generate`** / **`scene-compile --retime`** and **`manim`** instead. Preferred consumer order: narration → TTS → timestamps → scene-spec/compile → Manim → compose. - **Beat sync (fail-closed):** when `timing.json` has words, every story box label must match a spoken phrase (`wait_word`); unmatched labels and leftover LLM indices are rejected. Opt out with ``pace: none``. Fuzzy containment matching is not used. **`scene-compile` clamps FadeIn / page-fade `run_time` against the next word start** so `_TimedScene._clock` cannot race past waits (issue #66 — do not emit cascading first-board dumps). After a reveal, a **dwell** slot may play `Indicate` / `Circumscribe` when the gap to the next `wait_word` is long enough (also clamped). Long holds emit additional mid-hold pulses (`timed_wait` + emphasis) so the board does not freeze after the first Indicate. Optional box fields: `shape` (rounded/pill/diamond), `reveal` (fade/grow/slide), `emphasis` (none/pulse/ring). Page transitions FadeOut revealed boxes, not the parent `VGroup`. `scene-compile` refreshes stale `_box` / `_arrow` / `_TimedScene` helpers in `scenes.py`. - **Subject-beat coverage:** implemented in `scene_spec.layout_density_violations` / `cluster_subject_beats`; enforced by **`scene-spec-generate`** and **`validate`** (`validation.subject_beat_coverage.enabled`, default true). Not a blind label count. +- **`docgen benchmark` is required** after clock / compile / `_TimedScene` / dwell changes. Pytest string assertions are not a substitute. Do not remove the CI `benchmark` job. See **Required gate** below. - Prefer **stable CLI / library contracts** and **documented exit codes** so CI can depend on them. - **`narration_from_source`:** hints in config + **`docgen narration-generate`** — owner-supplied context paths, not opaque bulk edits to outputs. - Avoid duplicating long orchestration docs here; **link** to downstream repos when describing their publish pipelines. +## Required gate: `docgen benchmark` + +Clock, compile, scene-spec motion, `_TimedScene` helpers, and dwell/hold changes **must** stay green on the packaged corpus: + +1. Run **`docgen benchmark`** (or **`./scripts/benchmark-scenes.sh`**). Exit 0 vs ``src/docgen/benchmark_data/baseline.json``. +2. Keep **`tests/test_scene_benchmark.py`** in the default ``pytest tests/`` run. +3. Keep the **`benchmark`** job in **`.github/workflows/ci.yml`** (it must invoke ``docgen benchmark``). Do not delete or skip it. +4. **`--update-baseline`** only when the scorecard change is the point of the PR; review the JSON diff. Do not bump the baseline to hide a regression. +5. When production fails in a new way, add a case to ``standard_cases()`` and update the baseline in the same PR. + +String assertions on compiled ``scenes.py`` and ``simulate_reveal_timeline`` are not a substitute — the harness executes ``construct()`` on the real clock. + ## Testing (downstream relevance) Tests should cover **CLI-visible behavior** and contracts that adopters rely on: **`yaml-generate`**, **`scene-spec-generate`**, **`scene-compile`**, **`validate`**, **`compose`**, **`generate-all`**, **`pages`**, **`init`**, **`benchmark`**, **config** loading (`repo_root`, `env_file`), and package exports. Use small in-tree fixtures; this library does not ship a dogfood bundle. Clock / compile changes must keep ``docgen benchmark`` at or above ``benchmark_data/baseline.json`` — string assertions on ``scenes.py`` are not enough. @@ -77,7 +90,7 @@ Tests should cover **CLI-visible behavior** and contracts that adopters rely on: - **Virtualenv:** the project is installed editable into **`/workspace/.venv`** (created by the startup update script). Shells do **not** auto-activate it — run `. /workspace/.venv/bin/activate` (or prefix the venv path) before `docgen`, `pytest`, or `ruff`. The `docgen` console script lives at `/workspace/.venv/bin/docgen`. - **System deps are pre-baked in the VM snapshot** (not the update script): `ffmpeg` + `tesseract-ocr` (validation/compose/OCR), plus `build-essential`, `python3-dev`, `libcairo2-dev`, `libpango1.0-dev`, `pkg-config` (needed to build the `manim` extra's `manimpango`/`pycairo` wheels). If a fresh VM ever lacks these, reinstall via apt before `pip install`. -- **Standard commands** are in `README.md` / `pyproject.toml` / `.github/workflows/ci.yml`: lint `ruff check src/ tests/`; tests `pytest tests/ -v --tb=short`; the CI unit job also exports `PYTHONPATH=src` (not needed locally because of the editable install, but harmless). +- **Standard commands** are in `README.md` / `pyproject.toml` / `.github/workflows/ci.yml`: lint `ruff check src/ tests/`; tests `pytest tests/ -v --tb=short`; **required** `docgen benchmark` (CI job `benchmark`). The CI unit job also exports `PYTHONPATH=src` (not needed locally because of the editable install, but harmless). - **OpenAI-gated vs offline commands:** `tts`, `timestamps --engine whisper`, `image-generate`, `narration-generate`, `scene-spec-generate`, and `yaml-generate --llm` call OpenAI and need `OPENAI_API_KEY` (integration tests auto-skip without it). Fully offline: `init`, `scene-compile`, `manim`, `compose`, `validate`, `lint`, `pages`, `concat`, `yaml-generate` (no `--llm`), `timestamps` (default `local` engine), and `benchmark`. - **`scene-compile` gotcha:** paced specs (`wait_word`) need a `timing.json` entry for that stem (`docgen timestamps` after TTS). Prefer `scene-compile --retime` after fresh timestamps; for a fully offline smoke render, author rows without wait indices only if you accept unpaced reveals. - **No in-repo dogfood bundle:** exercise the pipeline against a scratch bundle (`docgen init /tmp/ --defaults` in a throwaway git dir). Do not hand-edit consumer generated assets (see `.cursor/rules/no-asset-edits.mdc`). diff --git a/README.md b/README.md index b0e3890..baadd6e 100644 --- a/README.md +++ b/README.md @@ -101,10 +101,11 @@ python3 -m venv .venv source .venv/bin/activate pip install -e ".[dev]" pytest -docgen benchmark # standard scene-timing corpus (no Manim / OpenAI) +docgen benchmark # required: scene-timing corpus vs committed baseline ``` -CI installs `ffmpeg` and `tesseract` via apt — see `.github/workflows/ci.yml`. +CI runs **`ruff`**, **`pytest`**, and a required **`benchmark`** job (`docgen benchmark`). +CI also installs `ffmpeg` and `tesseract` via apt for unit tests — see `.github/workflows/ci.yml`. **Roadmap:** [milestones/README.md](milestones/README.md). diff --git a/src/docgen/benchmark_data/README.md b/src/docgen/benchmark_data/README.md index 656e949..b6f4d6f 100644 --- a/src/docgen/benchmark_data/README.md +++ b/src/docgen/benchmark_data/README.md @@ -19,6 +19,11 @@ production failure we have already shipped by accident: `baseline.json` is the last accepted scorecard. A change that raises `defect_points` or drops `quality_points` fails the command (exit 1). +This command is a **required gate**: CI job `benchmark` in +`.github/workflows/ci.yml`, `tests/test_scene_benchmark.py` in the default +pytest run, and `.cursor/rules/docgen-benchmark.mdc` for agents. Do not +remove those hooks. + After an **intentional** improvement: ```bash diff --git a/tests/test_scene_benchmark.py b/tests/test_scene_benchmark.py index 74f36ba..17eb246 100644 --- a/tests/test_scene_benchmark.py +++ b/tests/test_scene_benchmark.py @@ -129,3 +129,25 @@ def test_packaged_baseline_exists() -> None: assert path.is_file() data = json.loads(path.read_text(encoding="utf-8")) assert set(data["cases"]) == {c.id for c in standard_cases()} + + +def test_cli_registers_benchmark_command() -> None: + assert "benchmark" in main.commands + + +def test_ci_workflow_requires_docgen_benchmark() -> None: + """Future PRs must not drop the named CI job that runs the corpus.""" + workflow = Path(__file__).resolve().parents[1] / ".github" / "workflows" / "ci.yml" + text = workflow.read_text(encoding="utf-8") + assert "\n benchmark:" in text + assert "docgen benchmark" in text + assert "Scene-timing benchmark" in text + + +def test_agent_rules_require_benchmark() -> None: + root = Path(__file__).resolve().parents[1] + agents = (root / "AGENTS.md").read_text(encoding="utf-8") + assert "## Required gate: `docgen benchmark`" in agents + rule = (root / ".cursor" / "rules" / "docgen-benchmark.mdc").read_text(encoding="utf-8") + assert "docgen benchmark" in rule + assert "alwaysApply: true" in rule