From 86cf915f25a429753ece6872150657d8be3687b3 Mon Sep 17 00:00:00 2001 From: 2002yy <15135142681@163.com> Date: Fri, 17 Jul 2026 19:40:12 +0800 Subject: [PATCH 1/2] add deterministic GitHub replay harness --- docs/PROJECT_STATUS.md | 63 ++++- src/web/github_pr_review_replay.py | 249 ++++++++++++++++++ .../github_pr_review_context_golden.json | 25 -- .../contexts/study-agent-pr28.json | 17 ++ .../contexts/study-agent-pr30.json | 30 +++ tests/fixtures/github_replay/manifest.json | 41 +++ tests/test_github_pr_review_evaluation.py | 14 +- tests/test_github_pr_review_replay.py | 110 ++++++++ tools/evaluate_github_replay.py | 32 +++ 9 files changed, 552 insertions(+), 29 deletions(-) create mode 100644 src/web/github_pr_review_replay.py delete mode 100644 tests/fixtures/github_pr_review_context_golden.json create mode 100644 tests/fixtures/github_replay/contexts/study-agent-pr28.json create mode 100644 tests/fixtures/github_replay/contexts/study-agent-pr30.json create mode 100644 tests/fixtures/github_replay/manifest.json create mode 100644 tests/test_github_pr_review_replay.py create mode 100644 tools/evaluate_github_replay.py diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index 51c1bacf..47b1fe08 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -2,8 +2,8 @@ > **唯一进度入口** > 更新：2026-07-17 -> 当前能力基线：PR #44 已合并，核心学习产品 G1–G8 已闭环，聊天联网工具循环已统一关联 durable ResearchRun -> 当前代码切片：恢复后的回答与 EvidenceTrail 同 Run 闭环及浏览器恢复路径已落地，下一步补慢查询阶段/停止取消/失败重试浏览器验收 +> 当前能力基线：PR #45 已合并，核心学习产品 G1–G8、聊天 ResearchRun 恢复/取消/EvidenceTrail 闭环及 G10-C2 cross-fork Provider 取证已进入 `main` +> 当前代码切片：G10-C3a replay harness 第一小批已落地 immutable manifest、确定性 CLI/汇总和 curated seed 来源隔离；下一步扩充真实 Provider 多仓库 corpus 这里只回答：**做到哪里、还差什么、下一步做什么**。 @@ -289,6 +289,11 @@ PR #30 已完成主链收口： | checks / jobs / logs | 分页基础完成 | ref-resolution 预算统一、artifacts、rerun attempt、日志分段、持久化缓存 | | 跨版本结构影响 | 双仓库图完成 | rename inference、AST edit、跨文件移动、真实仓库评测 | | PR review context | 双仓库取证完成 | 多仓库真实 replay corpus、symbol/CI association 质量指标 | +| RAG 索引一致性 | 基础完成 | server-owned RagRun、staging/active version、CAS 写租约、失败不激活、Chroma stale 清理已完成；还缺可恢复的逐文档摄取队列和 parser/chunker manifest | +| RAG 检索质量 | 初版完成 | BM25、向量、RRF hybrid、metadata filter、来源限额、可选 reranker 与 explainable debug 已完成；当前仅 6 条干净 fixture，缺真实学习语料与困难负例 | +| RAG 文档理解 | 基础完成 | Markdown/TXT/DOCX/PDF 纯文本摄取可用；缺标题/章节/表格/页区块等结构化解析、切块预览和扫描件/OCR 降级说明 | +| RAG 回答可信度 | 未完成 | 已有 citation-first context 和来源行号；缺回答级 citation precision/recall、groundedness、answerability/refusal 与 stale revision 评测 | +| KnowledgeBase 治理 | 初版完成 | 文档列表、稳定 document/revision identity、删除与索引版本已完成；缺 collection/scope、active revision、完整文档/聚焦检索策略和增量同步 | | 全量 mypy 零错误 | 未完成 | 增量门禁已阻止新增，后续应按模块逐步归零 | | TaskContract UI override | 已完成 | 按 Turn 一次性选择已接入；发送后清空，retry/continuation 不继承新 Turn override | | 本地 checkout | 未完成 | clone/fetch/checkout 和 worktree 隔离 | @@ -327,6 +332,59 @@ PR #30 已完成主链收口： 4. 可写 worktree、diff、回归和回滚。 5. 增量更新、缓存清理和磁盘预算。 +### 2026-07-17 开源对照审计：GitHub 仓库代理 + +对照 [OpenHands](https://github.com/OpenHands/OpenHands)、[SWE-agent](https://github.com/SWE-agent/SWE-agent)、[aider](https://github.com/Aider-AI/aider) 和 [Continue](https://github.com/continuedev/continue) 后，当前判断如下： + +1. **现有优势应保留**：immutable commit、cross-fork repository 归属、Provider 分页/共享预算、partial/uncertainty 降级和 EvidenceTrail，比通用代码代理更接近审计级只读取证。 +2. **最高优先级缺口是真实评测，不是更多接口**：当前 curated review-context label 很小，尚不能证明 symbol mapping、CI association 和 change-impact 在真实多仓库上的代表性质量。SWE-agent 的 benchmark/replay 与 aider 的长期代码编辑评测说明，执行能力扩张前必须先有不可回退的质量基线。 +3. **执行边界尚未建立**：当前 snapshot 明确不是 checkout；尚无受控工作目录、sandbox adapter、命令 schema、输出预算、进程/网络/磁盘限制和重启清理。OpenHands 将命令/文件动作放进独立 sandbox 并返回结构化 observation；该边界应先于任何可写代理。 +4. **任务意图不能替代执行授权**：TaskContract 继续负责学习/研究/项目目标；另建 `ExecutionPolicy` 负责 `allow / ask / exclude`。参考 Continue，读取默认允许，写入、安装依赖、联网命令和 shell 默认询问；headless 中无法确认的动作拒绝执行。 +5. **模式需要明确分层**：只读取证、只读 checkout、受控 test/lint/build、可写 worktree 是四个不同能力层，不应通过一个“项目模式”一次性全部开放。 + +#### G10 推荐门禁与顺序 + +1. **G10-C3a 真实 replay harness**：至少 6 个公开仓库、24–30 个 immutable case，覆盖 Python/TypeScript/Java、普通 PR、cross-fork、rename/delete、CI 失败和 Provider 截断；分别报告 symbol mapping 与 CI association precision/recall/F1、coverage、partial rate、请求数、延迟和缓存命中率。 + - **第一小批已完成**：新增 schema v1 manifest、immutable base/head SHA 校验、context 路径边界、唯一 case ID、语言/场景/provenance 元数据、确定性 CLI，以及 symbol/CI 微观与宏观指标、Provider status/partial/request/latency/cache 汇总。 + - PR #28/#30 已迁移为 2 个 `curated_unit_seed`；报告固定显示 `provider_replay_cases=0`，不会把人工单元样例伪装成真实 Provider replay。下一批仍需采集跨仓库、跨语言真实 context，达到代表性目标前不启用质量门禁。 +2. **G10-C3b Provider 证据补齐**：release、artifact metadata/按需下载，以及按 run attempt -> job -> step -> 时间窗口读取日志；所有新结果继续携带 repository、commit SHA、provider status 和 stop reason。 +3. **G10-D0 只读 RepositoryWorkspaceRun**：受控临时目录、immutable checkout、Docker sandbox 优先、显式不安全的 process fallback、取消/恢复/过期清理和资源预算；只允许 list/read/search/diff。 +4. **G10-D1 确定性命令执行**：仓库配置映射为结构化 `CommandSpec`，只开放声明过的 test/lint/build；保存 stdout/stderr、exit code、timeout、耗时和 artifact，不接受模型拼接任意 shell。 +5. **G10-D2 可写 worktree**：独立 worktree、写前基线、写后 diff/回归、一键回滚，禁止直接修改主 checkout；完成前不开放私有仓库自动执行。 + +### 2026-07-17 开源对照审计：RAG / 知识学习 + +对照 [RAGFlow](https://github.com/infiniflow/ragflow)、[Khoj](https://github.com/khoj-ai/khoj)、[AnythingLLM](https://github.com/Mintplex-Labs/anything-llm) 和 [Open WebUI](https://github.com/open-webui/open-webui) 后，当前实现不是“缺 RAG”，而是已经具备可信运行骨架、尚未形成可信质量闭环。 + +#### 已有能力与差距 + +1. **索引一致性是现有强项**：server-owned RagRun、稳定 document/revision ID、staging/active version、CAS 写租约、vector stage 失败不激活、append 替换旧 revision、删除与 Chroma stale 清理已经落地；不应重做旧路线图中的这些项目。 +2. **检索链路已过 MVP**：已有 BM25、local/backend vector、RRF hybrid、metadata filter、单来源 chunk 上限、重复文本抑制、可选 reranker、分阶段耗时和 score breakdown。短期不应以“再接一个 vector DB”作为质量工作替代品。 +3. **评测规模不足**：当前 checked-in corpus 只有 6 条干净查询，且预期全部命中；没有长文档、噪声 PDF/DOCX、中英混合、同名主题、多来源拼接、矛盾/过期资料、不可回答问题和 stale revision case，也没有 production embedding 的可选 replay。 +4. **只评检索，不评最终回答**：现有指标覆盖 source hit、Recall@K、MRR、nDCG 和 empty rate，但没有 citation precision/recall、引用片段是否支持具体 claim、groundedness、answerability/refusal、遗漏关键来源和旧 revision 泄漏。 +5. **摄取仍是纯文本级**：DOCX 只读 paragraph，PDF 依赖 pypdf 文本抽取并以页标记拼接；切块主要按空行和字符预算，未保留 heading/table/list/page block 等结构。RAGFlow 的结构化文档理解、模板化切块和 chunk 可视化说明，应先提升“输入质量”，再考虑 GraphRAG。 +6. **切块缺少产品化可见性**：没有 parser/chunker profile、最小 chunk 合并策略、chunk preview、解析警告与人工排除。Open WebUI 对过小 chunk 合并、完整文档与聚焦检索的分离说明，这些能力比继续增加固定 `max_chars` 参数更有产品价值。 +7. **知识作用域仍偏单索引**：已有 metadata filter 和文档列表，但缺显式 collection/workspace scope、active revision、按学习目标选择知识集合，以及“完整文档 / focused retrieval / tools-only”策略。AnythingLLM 的 workspace/thread 文档作用域与整文上下文回退、Khoj 的自定义知识代理提供了可参考的产品边界。 +8. **摄取运行缺少逐文档恢复体验**：请求内有 RagRun 和 stage，但上传仍是同步完成后返回；缺逐文档 queue/status、失败单项重试、跨页面继续观察与后台恢复。AnythingLLM 的逐文档 embedding 进度和可离开页面队列是更成熟的交互基线。 +9. **学习产品的差异化仍成立**：`RetrievalQueryPlan` 已能结合 objective、gap 和 pedagogy protocol 构造私有检索 query，回答上下文也保留引用；下一步应把检索证据用于“验证理解和暴露缺口”，而不是复制通用知识库聊天界面。 + +#### RAG 推荐门禁与顺序 + +1. **RAG-K1 真实学习语料质量基线，作为 PR #45 后的核心产品第一优先级**：至少 12 份真实文档、30–40 个查询，覆盖 Markdown/PDF/DOCX、中英混合、长文、重复/矛盾/过时版本、多来源问题和不可回答问题。保留 deterministic local 子集，并提供显式联网/付费的 production embedding replay。 +2. **RAG-K1 同时补回答级评测**：除 Recall/MRR/nDCG 外，新增 citation precision/recall、claim support、groundedness、answerability/refusal、source diversity、stale revision leakage、端到端延迟和 embedding/rerank 成本；第一轮只记录基线，之后才设置不可回退门禁。 +3. **RAG-K2 结构化摄取与切块**：引入 `ParserResult -> DocumentBlock -> Chunk`，保留 page/heading/paragraph/table/list identity、parser/chunker version 和 warnings；提供 Markdown heading、prose、PDF page/layout 等策略 profile、最小 chunk 合并与 chunk preview。扫描件/OCR 和多模态解析作为可选 adapter，失败必须显式降级，不能伪装为完整解析。 +4. **RAG-K3 KnowledgeBase domain**：增加 collection/scope、active revision、逐文档状态与重试、索引 manifest/磁盘统计；明确 `full_document / focused_retrieval / tools_only` 三种上下文策略，并由 TaskContract、文档长度、模型 context budget 与用户选择共同决定。 +5. **RAG-K4 教学可信闭环**：把 citation validation、已掌握点/当前缺口、证据披露级别和 follow-up query rewrite 接入 PedagogyTurnPlan；验证“不知道”等弱输入、跨轮指代和多跳学习问题是否检索到正确证据且不过度泄露答案。 +6. **RAG-K5 增量同步与外部连接器后置**：先完成本地文件 refresh/watch、content hash 去重和删除传播，再考虑 Notion/Drive/网页同步。GraphRAG、重型分布式检索、全量 OCR/多模态和更多 vector DB 均后置，除非 K1 评测证明它们解决了真实失败。 + +### 统一下一阶段顺序 + +1. **已完成**：PR #45 已以 merge commit `972e94e` 合并，当前工作分支从该 `main` 基线创建。 +2. **本切片推进**：G10-C3a replay harness 基础设施和 curated seed 已完成；继续以小批次补真实多仓库 Provider replay，达到 6 仓库/24–30 case 前只记录基线、不宣称代表性质量。 +3. G10-C3a 达到最低 corpus 覆盖后，回到核心学习产品，连续完成 RAG-K1 retrieval + answer faithfulness 基线和 RAG-K2 结构化摄取。 +4. 根据 K1 数据决定是否先做 RAG-K3，或补 G10-C3b release/artifact/日志定位。 +5. 只有上述只读质量门禁稳定后，才进入 G10-D0；G10-D2 可写代理、私有仓库自动执行、GraphRAG 和重型连接器继续后置。 + ## 5. 当前验证 核心学习产品最近完整门禁： @@ -349,6 +407,7 @@ PR #30 已完成主链收口： - 当前 G10-C2 第二切片：change-impact 每次先 compare 重解析 base/head，再按双 SHA 与完整预算复用；review-context 每次先取得 PR 证据，再按双 SHA、review/CI 证据指纹与预算复用。缓存/API/Provider 专项 32 passed，评论证据变化失效与跨重启命中均有回归。 - 当前 Provider 分页/跨 owner 切片：review thread comments 嵌套 cursor、共享预算耗尽、fork head checks 优先与 base fallback 均有回归；相关 review/provider 专项 15 passed，Ruff 增量通过。 - 当前 cross-fork change-impact 切片：PR review context 不再返回 unsupported，而是复用 commit-pinned PR comparison 生成 base/head 双仓库源码图；双仓库 snapshot 路由、repository 归属、同 SHA 不同 fork 缓存隔离与 review-context 接线均有回归。聚焦测试 11 passed，GitHub 专项 94 passed，全量 pytest 777 passed，Ruff 全量通过；expanded mypy 当前 126 个既有错误，低于 127 基线且无新增。 +- 当前 G10-C3a replay harness 第一小批：PR #28/#30 的 immutable SHA 与 curated context 已进入 schema v1 manifest；CLI 两次输出字节级一致，明确报告 1 个仓库、2 个 seed、0 个 Provider replay。原有独立 golden JSON 已删除，manifest 成为唯一 label 真值；评测/replay 聚焦 7 passed，GitHub 专项 97 passed，全量 pytest 781 passed，Ruff 全量通过，新增模块目标 mypy 通过。因 C 盘空间不足，全量 pytest 的临时目录改到 D 盘后通过。 PR #31 功能代码验证： diff --git a/src/web/github_pr_review_replay.py b/src/web/github_pr_review_replay.py new file mode 100644 index 00000000..69c08302 --- /dev/null +++ b/src/web/github_pr_review_replay.py @@ -0,0 +1,249 @@ +"""Deterministic manifest runner for recorded GitHub PR review-context replays.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from src.web.github_pr_review_evaluation import evaluate_pr_review_context + + +_SHA_PATTERN = re.compile(r"^[0-9a-f]{40}$") +_REPOSITORY_PATTERN = re.compile(r"^[^/\s]+/[^/\s]+$") + + +@dataclass(frozen=True) +class GitHubReplayCase: + case_id: str + repository: str + pull_request: int + base_sha: str + head_sha: str + language: str + scenarios: tuple[str, ...] + context_path: Path + expected: dict[str, Any] + provenance: str + provider_replay: bool + + +@dataclass(frozen=True) +class GitHubReplayManifest: + corpus_id: str + schema_version: int + cases: tuple[GitHubReplayCase, ...] + + +def _load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def _required_string(value: Any, field: str) -> str: + resolved = str(value or "").strip() + if not resolved: + raise ValueError(f"GitHub replay case requires {field}") + return resolved + + +def _string_tuple(value: Any, field: str, *, allow_empty: bool = False) -> tuple[str, ...]: + if not isinstance(value, list): + raise ValueError(f"GitHub replay case {field} must be a list") + resolved = tuple(str(item).strip() for item in value if str(item).strip()) + if not resolved and not allow_empty: + raise ValueError(f"GitHub replay case requires {field}") + return resolved + + +def _context_path(manifest_path: Path, value: Any) -> Path: + relative = Path(_required_string(value, "context_path")) + if relative.is_absolute() or ".." in relative.parts: + raise ValueError("GitHub replay context_path must stay under the manifest directory") + root = manifest_path.parent.resolve() + resolved = (root / relative).resolve() + if root != resolved and root not in resolved.parents: + raise ValueError("GitHub replay context_path escapes the manifest directory") + if not resolved.is_file(): + raise ValueError(f"GitHub replay context not found: {relative.as_posix()}") + return resolved + + +def _parse_case(manifest_path: Path, value: Any) -> GitHubReplayCase: + if not isinstance(value, dict): + raise ValueError("GitHub replay cases must be objects") + repository = _required_string(value.get("repository"), "repository") + if not _REPOSITORY_PATTERN.fullmatch(repository): + raise ValueError(f"Invalid GitHub replay repository: {repository}") + base_sha = _required_string(value.get("base_sha"), "base_sha").lower() + head_sha = _required_string(value.get("head_sha"), "head_sha").lower() + if not _SHA_PATTERN.fullmatch(base_sha) or not _SHA_PATTERN.fullmatch(head_sha): + raise ValueError("GitHub replay base_sha and head_sha must be immutable 40-char SHAs") + pull_request = int(value.get("pull_request") or 0) + if pull_request <= 0: + raise ValueError("GitHub replay pull_request must be positive") + labels = value.get("labels") + if not isinstance(labels, dict): + raise ValueError("GitHub replay case requires labels") + case_id = _required_string(value.get("case_id"), "case_id") + expected: dict[str, Any] = { + "case_id": case_id, + "repository": repository, + "pull_request": pull_request, + "review_symbol_ids": list( + _string_tuple(labels.get("review_symbol_ids"), "review_symbol_ids", allow_empty=True) + ), + "ci_test_paths": list( + _string_tuple(labels.get("ci_test_paths"), "ci_test_paths", allow_empty=True) + ), + } + return GitHubReplayCase( + case_id=case_id, + repository=repository, + pull_request=pull_request, + base_sha=base_sha, + head_sha=head_sha, + language=_required_string(value.get("language"), "language").lower(), + scenarios=_string_tuple(value.get("scenarios"), "scenarios"), + context_path=_context_path(manifest_path, value.get("context_path")), + expected=expected, + provenance=_required_string(value.get("provenance"), "provenance"), + provider_replay=bool(value.get("provider_replay", False)), + ) + + +def load_github_replay_manifest(path: str | Path) -> GitHubReplayManifest: + manifest_path = Path(path).resolve() + payload = _load_json(manifest_path) + if not isinstance(payload, dict): + raise ValueError("GitHub replay manifest must be an object") + schema_version = int(payload.get("schema_version") or 0) + if schema_version != 1: + raise ValueError(f"Unsupported GitHub replay schema_version: {schema_version}") + raw_cases = payload.get("cases") + if not isinstance(raw_cases, list) or not raw_cases: + raise ValueError("GitHub replay manifest requires cases") + cases = tuple(_parse_case(manifest_path, value) for value in raw_cases) + case_ids = [case.case_id for case in cases] + if len(case_ids) != len(set(case_ids)): + raise ValueError("GitHub replay case_id values must be unique") + return GitHubReplayManifest( + corpus_id=_required_string(payload.get("corpus_id"), "corpus_id"), + schema_version=schema_version, + cases=cases, + ) + + +def _ratio(numerator: int, denominator: int, *, empty: float = 0.0) -> float: + return round(numerator / denominator, 4) if denominator else empty + + +def _aggregate_metric(results: list[dict[str, Any]], key: str) -> dict[str, Any]: + metrics = [dict(result[key]) for result in results] + true_positive = sum(int(item["true_positive"]) for item in metrics) + false_positive = sum(int(item["false_positive"]) for item in metrics) + false_negative = sum(int(item["false_negative"]) for item in metrics) + precision = _ratio(true_positive, true_positive + false_positive, empty=1.0) + recall = _ratio(true_positive, true_positive + false_negative, empty=1.0) + f1 = ( + round(2 * precision * recall / (precision + recall), 4) + if precision + recall + else 0.0 + ) + return { + "true_positive": true_positive, + "false_positive": false_positive, + "false_negative": false_negative, + "precision": precision, + "recall": recall, + "f1": f1, + } + + +def _mean(values: list[float]) -> float: + return round(sum(values) / len(values), 3) if values else 0.0 + + +def evaluate_github_replay_manifest(path: str | Path) -> dict[str, Any]: + manifest = load_github_replay_manifest(path) + case_results: list[dict[str, Any]] = [] + provider_statuses: dict[str, int] = {} + requests: list[float] = [] + elapsed: list[float] = [] + cache_hits = 0 + + for case in manifest.cases: + context = _load_json(case.context_path) + if not isinstance(context, dict): + raise ValueError(f"GitHub replay context must be an object: {case.case_id}") + metrics = evaluate_pr_review_context(context, case.expected) + replay_metadata = context.get("replay_metadata") + metadata = dict(replay_metadata) if isinstance(replay_metadata, dict) else {} + provider_status = str(context.get("provider_status") or "unknown") + provider_statuses[provider_status] = provider_statuses.get(provider_status, 0) + 1 + requests.append(float(metadata.get("provider_requests") or 0)) + elapsed.append(float(metadata.get("elapsed_ms") or 0)) + cache_hit = bool(metadata.get("cache_hit", False)) + cache_hits += int(cache_hit) + case_results.append( + { + "case_id": case.case_id, + "repository": case.repository, + "pull_request": case.pull_request, + "base_sha": case.base_sha, + "head_sha": case.head_sha, + "language": case.language, + "scenarios": list(case.scenarios), + "provenance": case.provenance, + "provider_replay": case.provider_replay, + "provider_status": provider_status, + "provider_requests": int(requests[-1]), + "elapsed_ms": round(elapsed[-1], 3), + "cache_hit": cache_hit, + "metrics": metrics, + } + ) + + metric_results = [dict(item["metrics"]) for item in case_results] + symbol = _aggregate_metric(metric_results, "review_symbol_mapping") + ci = _aggregate_metric(metric_results, "failed_ci_test_association") + total = len(case_results) + provider_replays = sum(int(item["provider_replay"]) for item in case_results) + partial = provider_statuses.get("partial", 0) + repositories = sorted({case.repository for case in manifest.cases}) + languages = sorted({case.language for case in manifest.cases}) + scenarios = sorted({scenario for case in manifest.cases for scenario in case.scenarios}) + return { + "corpus_id": manifest.corpus_id, + "schema_version": manifest.schema_version, + "coverage": { + "cases": total, + "repositories": len(repositories), + "repository_names": repositories, + "languages": languages, + "scenarios": scenarios, + "provider_replay_cases": provider_replays, + "curated_seed_cases": total - provider_replays, + }, + "provider": { + "status_counts": dict(sorted(provider_statuses.items())), + "partial_rate": _ratio(partial, total), + "mean_requests": _mean(requests), + "mean_elapsed_ms": _mean(elapsed), + "cache_hit_rate": _ratio(cache_hits, total), + }, + "metrics": { + "review_symbol_mapping": symbol, + "failed_ci_test_association": ci, + "macro": { + "precision": round((symbol["precision"] + ci["precision"]) / 2, 4), + "recall": round((symbol["recall"] + ci["recall"]) / 2, 4), + "f1": round((symbol["f1"] + ci["f1"]) / 2, 4), + "mean_case_f1": _mean( + [float(item["metrics"]["macro"]["f1"]) for item in case_results] + ), + }, + }, + "cases": case_results, + } diff --git a/tests/fixtures/github_pr_review_context_golden.json b/tests/fixtures/github_pr_review_context_golden.json deleted file mode 100644 index 5b41a77d..00000000 --- a/tests/fixtures/github_pr_review_context_golden.json +++ /dev/null @@ -1,25 +0,0 @@ -[ - { - "case_id": "study-agent-pr30-task-contract", - "repository": "2002yy/study-agent", - "pull_request": 30, - "review_symbol_ids": [ - "symbol-resolve-turn-task-contract" - ], - "ci_test_paths": [ - "tests/test_task_contract.py", - "tests/test_external_data_policy.py" - ], - "notes": "Curated replay labels for the TaskContract propagation slice. Provider calls are not part of the unit test." - }, - { - "case_id": "study-agent-pr28-change-impact", - "repository": "2002yy/study-agent", - "pull_request": 28, - "review_symbol_ids": [], - "ci_test_paths": [ - "tests/test_github_change_impact.py" - ], - "notes": "Curated replay labels for version-aware GitHub change impact. Empty review labels verify no fabricated mapping." - } -] diff --git a/tests/fixtures/github_replay/contexts/study-agent-pr28.json b/tests/fixtures/github_replay/contexts/study-agent-pr28.json new file mode 100644 index 00000000..ed3e3304 --- /dev/null +++ b/tests/fixtures/github_replay/contexts/study-agent-pr28.json @@ -0,0 +1,17 @@ +{ + "provider_status": "curated", + "replay_metadata": { + "provider_requests": 0, + "elapsed_ms": 0, + "cache_hit": false + }, + "review_items": [], + "ci_associations": [ + { + "association": { + "status": "associated", + "tests": ["tests/test_github_change_impact.py"] + } + } + ] +} diff --git a/tests/fixtures/github_replay/contexts/study-agent-pr30.json b/tests/fixtures/github_replay/contexts/study-agent-pr30.json new file mode 100644 index 00000000..efcce36e --- /dev/null +++ b/tests/fixtures/github_replay/contexts/study-agent-pr30.json @@ -0,0 +1,30 @@ +{ + "provider_status": "curated", + "replay_metadata": { + "provider_requests": 0, + "elapsed_ms": 0, + "cache_hit": false + }, + "review_items": [ + { + "mapping": { + "status": "mapped", + "symbol": { + "identity": {"id": "symbol-resolve-turn-task-contract"}, + "qualified_name": "resolve_turn_task_contract" + } + } + } + ], + "ci_associations": [ + { + "association": { + "status": "associated", + "tests": [ + "tests/test_task_contract.py", + "tests/test_external_data_policy.py" + ] + } + } + ] +} diff --git a/tests/fixtures/github_replay/manifest.json b/tests/fixtures/github_replay/manifest.json new file mode 100644 index 00000000..1a1a0100 --- /dev/null +++ b/tests/fixtures/github_replay/manifest.json @@ -0,0 +1,41 @@ +{ + "corpus_id": "study-agent-g10-c3a-seed-v1", + "schema_version": 1, + "cases": [ + { + "case_id": "study-agent-pr30-task-contract", + "repository": "2002yy/study-agent", + "pull_request": 30, + "base_sha": "c94263e48d7f675bd328b8eb42ca2e2f83d2f4ba", + "head_sha": "84c7c97798786b57cbc570e57daf74816f2af475", + "language": "python", + "scenarios": ["review-symbol", "failed-ci-test"], + "context_path": "contexts/study-agent-pr30.json", + "provenance": "curated_unit_seed", + "provider_replay": false, + "labels": { + "review_symbol_ids": ["symbol-resolve-turn-task-contract"], + "ci_test_paths": [ + "tests/test_task_contract.py", + "tests/test_external_data_policy.py" + ] + } + }, + { + "case_id": "study-agent-pr28-change-impact", + "repository": "2002yy/study-agent", + "pull_request": 28, + "base_sha": "f4e33cc29b7f7616521697d42343624b4f5d1760", + "head_sha": "68a0eefd2e0ad68b3dda43e77e12235184894a3b", + "language": "python", + "scenarios": ["empty-review-label", "change-impact-test"], + "context_path": "contexts/study-agent-pr28.json", + "provenance": "curated_unit_seed", + "provider_replay": false, + "labels": { + "review_symbol_ids": [], + "ci_test_paths": ["tests/test_github_change_impact.py"] + } + } + ] +} diff --git a/tests/test_github_pr_review_evaluation.py b/tests/test_github_pr_review_evaluation.py index 3a310ab6..853556d5 100644 --- a/tests/test_github_pr_review_evaluation.py +++ b/tests/test_github_pr_review_evaluation.py @@ -5,11 +5,21 @@ from src.web.github_pr_review_evaluation import evaluate_pr_review_context -_FIXTURE = Path(__file__).parent / "fixtures" / "github_pr_review_context_golden.json" +_FIXTURE = Path(__file__).parent / "fixtures" / "github_replay" / "manifest.json" def _cases() -> list[dict]: - return json.loads(_FIXTURE.read_text(encoding="utf-8")) + payload = json.loads(_FIXTURE.read_text(encoding="utf-8")) + return [ + { + "case_id": case["case_id"], + "repository": case["repository"], + "pull_request": case["pull_request"], + "review_symbol_ids": case["labels"]["review_symbol_ids"], + "ci_test_paths": case["labels"]["ci_test_paths"], + } + for case in payload["cases"] + ] def test_review_context_metrics_report_exact_precision_and_recall(): diff --git a/tests/test_github_pr_review_replay.py b/tests/test_github_pr_review_replay.py new file mode 100644 index 00000000..ef1c7a76 --- /dev/null +++ b/tests/test_github_pr_review_replay.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from src.web.github_pr_review_replay import ( + evaluate_github_replay_manifest, + load_github_replay_manifest, +) + + +_FIXTURE_DIR = Path(__file__).parent / "fixtures" / "github_replay" +_MANIFEST = _FIXTURE_DIR / "manifest.json" + + +def test_manifest_loads_immutable_seed_cases_without_claiming_provider_replay(): + manifest = load_github_replay_manifest(_MANIFEST) + + assert manifest.corpus_id == "study-agent-g10-c3a-seed-v1" + assert len(manifest.cases) == 2 + assert all(len(case.base_sha) == 40 and len(case.head_sha) == 40 for case in manifest.cases) + assert all(case.provenance == "curated_unit_seed" for case in manifest.cases) + assert not any(case.provider_replay for case in manifest.cases) + + +def test_manifest_summary_reports_quality_coverage_and_provider_limits(): + summary = evaluate_github_replay_manifest(_MANIFEST) + + assert summary["coverage"] == { + "cases": 2, + "repositories": 1, + "repository_names": ["2002yy/study-agent"], + "languages": ["python"], + "scenarios": [ + "change-impact-test", + "empty-review-label", + "failed-ci-test", + "review-symbol", + ], + "provider_replay_cases": 0, + "curated_seed_cases": 2, + } + assert summary["provider"] == { + "status_counts": {"curated": 2}, + "partial_rate": 0.0, + "mean_requests": 0.0, + "mean_elapsed_ms": 0.0, + "cache_hit_rate": 0.0, + } + assert summary["metrics"]["macro"] == { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "mean_case_f1": 1.0, + } + + +def test_manifest_rejects_mutable_ref_and_context_path_escape(tmp_path: Path): + context = tmp_path / "context.json" + context.write_text("{}", encoding="utf-8") + payload = json.loads(_MANIFEST.read_text(encoding="utf-8")) + payload["cases"] = [dict(payload["cases"][0])] + payload["cases"][0]["base_sha"] = "main" + payload["cases"][0]["context_path"] = "context.json" + manifest = tmp_path / "manifest.json" + manifest.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="immutable 40-char SHAs"): + load_github_replay_manifest(manifest) + + payload["cases"][0]["base_sha"] = "a" * 40 + payload["cases"][0]["context_path"] = "../outside.json" + manifest.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValueError, match="must stay under"): + load_github_replay_manifest(manifest) + + +def test_replay_cli_writes_deterministic_json(tmp_path: Path): + output = tmp_path / "summary.json" + completed = subprocess.run( + [ + sys.executable, + "tools/evaluate_github_replay.py", + str(_MANIFEST), + "--output", + str(output), + ], + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode == 0, completed.stderr + first = output.read_text(encoding="utf-8") + assert json.loads(first)["coverage"]["provider_replay_cases"] == 0 + subprocess.run( + [ + sys.executable, + "tools/evaluate_github_replay.py", + str(_MANIFEST), + "--output", + str(output), + ], + check=True, + ) + assert output.read_text(encoding="utf-8") == first diff --git a/tools/evaluate_github_replay.py b/tools/evaluate_github_replay.py new file mode 100644 index 00000000..5421e6e2 --- /dev/null +++ b/tools/evaluate_github_replay.py @@ -0,0 +1,32 @@ +"""Evaluate a checked-in GitHub review-context replay manifest.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from src.web.github_pr_review_replay import evaluate_github_replay_manifest + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("manifest", type=Path) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + result = evaluate_github_replay_manifest(args.manifest) + rendered = json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered, encoding="utf-8") + else: + sys.stdout.write(rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 7b6473472ddf6b01bdfcd9f6666c62e4e336e8d7 Mon Sep 17 00:00:00 2001 From: 2002yy <15135142681@163.com> Date: Fri, 17 Jul 2026 19:44:29 +0800 Subject: [PATCH 2/2] allow immutable git SHAs in secret scan --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af1b1106..371a625a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,7 @@ jobs: pip install detect-secrets 2>/dev/null detect-secrets scan --disable-plugin KeywordDetector \ --exclude-files '.*\.(pyc|jpg|png|zip)$' \ + --exclude-lines '"(base_sha|head_sha)": "[0-9a-f]{40}"' \ .github README.md docs src tests tools config templates roles changelog assets frontend .env.example \ > detect-secrets-report.json 2>&1 python - <<'PY'