diff --git a/src/hyrule_engineering_loop/backend.py b/src/hyrule_engineering_loop/backend.py index ba44409..100021e 100644 --- a/src/hyrule_engineering_loop/backend.py +++ b/src/hyrule_engineering_loop/backend.py @@ -580,7 +580,96 @@ def _parse_harness_output(self, stdout: str) -> dict[str, Any]: decoded = json.loads(stdout) except (json.JSONDecodeError, ValueError): return {} - return decoded if isinstance(decoded, dict) else {} + if not isinstance(decoded, dict): + return {} + + parsed = dict(decoded) + + def number_from(mapping: Mapping[str, Any], *keys: str) -> float | None: + for key in keys: + value = mapping.get(key) + if isinstance(value, (int, float)) and not isinstance(value, bool): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + continue + return None + + raw_usage = parsed.get("usage") + normalized_usage = dict(raw_usage) if isinstance(raw_usage, dict) else {} + input_tokens = number_from( + normalized_usage, + "input_tokens", + "inputTokens", + "prompt_tokens", + "promptTokens", + ) + if input_tokens is None: + input_tokens = number_from( + parsed, "input_tokens", "inputTokens", "prompt_tokens", "promptTokens" + ) + output_tokens = number_from( + normalized_usage, + "output_tokens", + "outputTokens", + "completion_tokens", + "completionTokens", + ) + if output_tokens is None: + output_tokens = number_from( + parsed, + "output_tokens", + "outputTokens", + "completion_tokens", + "completionTokens", + ) + if input_tokens is not None: + normalized_usage["input_tokens"] = int(input_tokens) + if output_tokens is not None: + normalized_usage["output_tokens"] = int(output_tokens) + if normalized_usage: + parsed["usage"] = normalized_usage + + cost = number_from(parsed, "total_cost_usd", "totalCostUsd", "cost_usd", "costUsd") + raw_cost = parsed.get("cost") + if cost is None and isinstance(raw_cost, dict): + cost = number_from(raw_cost, "usd", "total_usd", "totalCostUsd", "cost_usd") + if cost is not None: + parsed["total_cost_usd"] = float(cost) + + turns = number_from(parsed, "num_turns", "numTurns", "turns") + if turns is not None: + parsed["num_turns"] = int(turns) + + if "is_error" not in parsed: + parsed["is_error"] = parsed.get("type") == "error" or bool(parsed.get("error")) + + result = parsed.get("result") + if isinstance(result, dict): + text = next( + ( + str(result[key]) + for key in ("text", "content", "message", "summary") + if isinstance(result.get(key), str) + ), + "", + ) + parsed["result"] = text + elif not isinstance(result, str): + text = next( + ( + str(parsed[key]) + for key in ("output", "text", "content", "message") + if isinstance(parsed.get(key), str) + ), + "", + ) + if text: + parsed["result"] = text + + return parsed def execute( self, @@ -722,7 +811,7 @@ class PiBackend(SubprocessBackend): """ name = "pi" - default_command = ("pi", "--print", "{prompt}") + default_command = ("pi", "--print", "--mode", "json", "{prompt}") extra_env_names = PI_PROVIDER_ENV_NAMES diff --git a/src/hyrule_engineering_loop/daemon.py b/src/hyrule_engineering_loop/daemon.py index e6ee7a0..c49f088 100644 --- a/src/hyrule_engineering_loop/daemon.py +++ b/src/hyrule_engineering_loop/daemon.py @@ -18,6 +18,7 @@ import json import os +import re import ssl import time import urllib.request @@ -93,6 +94,31 @@ } HIGH_RISK_LABELS = frozenset({"critical", "security"}) +BUDGET_LABEL_TIERS: dict[str, str] = { + "loop:budget-s": "s", + "loop:budget-small": "s", + "loop:budget-m": "m", + "loop:budget-medium": "m", + "loop:budget-l": "l", + "loop:budget-large": "l", + "loop:budget-xl": "xl", + "loop:budget-extra-large": "xl", +} +BUDGET_BODY_RE = re.compile( + r"(?im)^\s*(?:loop[-_ ]?budget|budget)\s*:\s*" + r"(?Ps|small|m|medium|l|large|xl|extra[-_ ]?large)\s*$" +) +BUDGET_TIER_MULTIPLIERS: dict[str, float] = { + "s": 1.0, + "m": 1.0, + "l": 2.0, + "xl": 4.0, +} +BUDGET_TIER_FLOORS: dict[str, tuple[int, int, float]] = { + "l": (40, 90, 10.0), + "xl": (80, 180, 20.0), +} + class DaemonError(RuntimeError): """Raised when a daemon cycle cannot run at all.""" @@ -599,6 +625,68 @@ def _run_cost(final_state: dict[str, Any]) -> float: ) +def _normalize_budget_tier(value: str) -> str | None: + normalized = value.strip().lower().replace("_", "-").replace(" ", "-") + if normalized in {"s", "small"}: + return "s" + if normalized in {"m", "medium"}: + return "m" + if normalized in {"l", "large"}: + return "l" + if normalized in {"xl", "extra-large"}: + return "xl" + return None + + +def issue_budget_tier(item: IntakeItem, *, body: str) -> str | None: + """Resolve the requested per-issue budget tier from labels or body fields.""" + for label in item.labels: + tier = BUDGET_LABEL_TIERS.get(label.strip().lower()) + if tier is not None: + return tier + match = BUDGET_BODY_RE.search(body) + if match is None: + return None + return _normalize_budget_tier(match.group("tier")) + + +def backend_budget_for_issue( + config: DaemonConfig, + item: IntakeItem, + *, + body: str, +) -> dict[str, Any]: + """Build the backend budget for one issue, including explicit sizing signals.""" + budget: dict[str, Any] = { + "max_iterations": config.max_iterations_per_run, + "max_wall_clock_minutes": config.max_wall_clock_minutes_per_run, + "max_cost_usd": config.max_cost_usd_per_run, + "tier": "default", + } + tier = issue_budget_tier(item, body=body) + if tier is None: + return budget + + budget["tier"] = tier + multiplier = BUDGET_TIER_MULTIPLIERS[tier] + budget["max_iterations"] = max(1, int(config.max_iterations_per_run * multiplier)) + budget["max_wall_clock_minutes"] = max( + 1, + int(config.max_wall_clock_minutes_per_run * multiplier), + ) + budget["max_cost_usd"] = round(config.max_cost_usd_per_run * multiplier, 4) + + floor = BUDGET_TIER_FLOORS.get(tier) + if floor is not None: + min_iterations, min_minutes, min_cost = floor + budget["max_iterations"] = max(budget["max_iterations"], min_iterations) + budget["max_wall_clock_minutes"] = max( + budget["max_wall_clock_minutes"], min_minutes + ) + budget["max_cost_usd"] = max(float(budget["max_cost_usd"]), min_cost) + return budget + + # --- the cycle -------------------------------------------------------------- @@ -767,11 +855,7 @@ def daemon_once( allowed_paths=effective_allowed_paths, source_files=["README.md"], memory_dir=config.memory_dir, - backend_budget={ - "max_iterations": config.max_iterations_per_run, - "max_wall_clock_minutes": config.max_wall_clock_minutes_per_run, - "max_cost_usd": config.max_cost_usd_per_run, - }, + backend_budget=backend_budget_for_issue(config, item, body=body), knowledge_context=config.knowledge_context, knowledge_learning_dir=config.knowledge_learning_dir, ) diff --git a/src/hyrule_engineering_loop/gate_runner.py b/src/hyrule_engineering_loop/gate_runner.py index b3c324b..7963ad4 100644 --- a/src/hyrule_engineering_loop/gate_runner.py +++ b/src/hyrule_engineering_loop/gate_runner.py @@ -5,6 +5,7 @@ import json import subprocess import sys +import tomllib from pathlib import Path from typing import Any, Iterable, Sequence @@ -45,6 +46,8 @@ def run_gate_commands( if not argv: raise ValueError("gate command cannot be empty") + startup_error = False + result: dict[str, Any] try: completed = subprocess.run( argv, @@ -67,6 +70,16 @@ def run_gate_commands( "stdout": _clip(_as_text(exc.stdout)), "stderr": _clip(_as_text(exc.stderr) or f"timed out after {timeout_seconds}s"), } + except OSError as exc: + startup_error = True + result = { + "command": argv, + "returncode": 127, + "stdout": "", + "stderr": _clip(f"could not start command: {exc}"), + } + + result["status"] = "passed" if result["returncode"] == 0 else "failed" results.append(result) if result["returncode"] != 0: @@ -74,7 +87,11 @@ def run_gate_commands( { "node": "gate_execution", "domain": "ci", - "message": f"command failed: {' '.join(argv)}", + "message": ( + f"command could not start: {' '.join(argv)}" + if startup_error + else f"command failed: {' '.join(argv)}" + ), "returncode": result["returncode"], "stderr": result["stderr"], } @@ -83,12 +100,43 @@ def run_gate_commands( return results, errors -def select_gate_commands_for_mutations(paths: Iterable[str]) -> list[list[str]]: +def _python_project_gate_prefix(cwd: Path | str | None) -> list[str] | None: + if cwd is None: + return None + pyproject = Path(cwd) / "pyproject.toml" + if not pyproject.is_file(): + return None + try: + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) + except tomllib.TOMLDecodeError: + return None + project = data.get("project") + optional = project.get("optional-dependencies") if isinstance(project, dict) else None + if isinstance(optional, dict) and "dev" in optional: + return ["uv", "run", "--extra", "dev"] + groups = data.get("dependency-groups") + if isinstance(groups, dict) and "dev" in groups: + return ["uv", "run", "--group", "dev"] + return None + + +def select_gate_commands_for_mutations( + paths: Iterable[str], + *, + cwd: Path | str | None = None, +) -> list[list[str]]: """Select local, workspace-safe gates from proposed mutation paths.""" normalized = [path.split(":", 1)[1] if ":" in path else path for path in paths] if not normalized: return [] if any(path.endswith(".py") for path in normalized): + prefix = _python_project_gate_prefix(cwd) + if prefix is not None: + return [ + [*prefix, "ruff", "check", "."], + [*prefix, "mypy", "."], + [*prefix, "python", "-m", "pytest", "-q"], + ] return [[sys.executable, "-m", "compileall", "-q", "."]] if all(path.startswith("docs/") or path.endswith((".md", ".txt", ".rst")) for path in normalized): paths_literal = repr(json.dumps(normalized)) diff --git a/src/hyrule_engineering_loop/nodes.py b/src/hyrule_engineering_loop/nodes.py index e0388a7..2cbade2 100644 --- a/src/hyrule_engineering_loop/nodes.py +++ b/src/hyrule_engineering_loop/nodes.py @@ -4,6 +4,7 @@ import hashlib import json +import re from pathlib import Path from typing import Any, Iterable, cast @@ -64,6 +65,21 @@ # unchanged worktree diff (the Phase F no-progress kill criterion). STALL_ROUND_LIMIT = 3 +DEFAULT_ACCEPTANCE_CRITERIA: tuple[str, ...] = ( + "The request is implemented within the allowed paths of each target repo.", + "All selected gates pass in the branch-backed worktree.", + "The diff introduces no secret material or denied content patterns.", +) + +ACCEPTANCE_HEADING_RE = re.compile( + r"^\s{0,3}#{1,6}\s+acceptance(?:\s+criteria)?\s*$", + re.IGNORECASE, +) +MARKDOWN_HEADING_RE = re.compile(r"^\s{0,3}#{1,6}\s+\S+") +LIST_ITEM_RE = re.compile( + r"^\s*(?:[-*+]\s+(?:\[[ xX]\]\s*)?|\d+[.)]\s+)(?P.+?)\s*$" +) + ALL_ROLES: tuple[RoleName, ...] = ( "network_architect", "systems_engineer", @@ -82,6 +98,46 @@ "virtual_lab_chaos": "virtual_lab_chaos", } + +def _acceptance_criteria_from_request(request: str) -> list[str]: + """Extract deterministic acceptance criteria from an issue/request body.""" + section_lines: list[str] = [] + in_section = False + for line in request.splitlines(): + if ACCEPTANCE_HEADING_RE.match(line): + in_section = True + continue + if in_section and MARKDOWN_HEADING_RE.match(line): + break + if in_section: + section_lines.append(line) + + criteria: list[str] = [] + for raw_line in section_lines: + line = raw_line.strip() + if not line: + continue + match = LIST_ITEM_RE.match(line) + if match: + text = match.group("text").strip() + if text: + criteria.append(text) + continue + if criteria and raw_line.startswith((" ", "\t")): + criteria[-1] = f"{criteria[-1]} {line}" + else: + criteria.append(line) + return criteria + + +def _gate_selection_cwd(state: GraphState) -> Path | None: + for worktree in state.get("worktree_results") or []: + path = worktree.get("worktree_path") if isinstance(worktree, dict) else None + if path: + return Path(str(path)) + root = state.get("workspace_root") + return Path(str(root)) if root else None + ROLE_PROMPT_FILES_HINT: dict[RoleName, str] = { "network_architect": "role-network-architect/SKILL.md", "systems_engineer": "role-systems-engineer/SKILL.md", @@ -320,6 +376,9 @@ def planner_node(state: GraphState) -> StateUpdate: (line.strip() for line in request.splitlines() if line.strip()), "(no request text supplied)", )[:300] + acceptance_criteria = _acceptance_criteria_from_request(request) or list( + DEFAULT_ACCEPTANCE_CRITERIA + ) spec = { "change_id": state["change_id"], "change_class": str(state["change_class"]), @@ -335,11 +394,7 @@ def planner_node(state: GraphState) -> StateUpdate: "budget": dict(state.get("backend_budget") or DEFAULT_BUDGET), "intake_source": "operator", "intent": intent, - "acceptance_criteria": [ - "The request is implemented within the allowed paths of each target repo.", - "All selected gates pass in the branch-backed worktree.", - "The diff introduces no secret material or denied content patterns.", - ], + "acceptance_criteria": acceptance_criteria, "non_goals": "Anything outside the allowed paths; unrelated refactors.", "rollback_sketch": state.get("rollback_plan") or "Discard the generated worktree and branch; no production state changes.", @@ -791,7 +846,8 @@ def delegate_implementation_node(state: GraphState) -> StateUpdate: ) if not state.get("gate_commands") and (changed_paths or mutations): update["gate_commands"] = select_gate_commands_for_mutations( - changed_paths or list(mutations) + changed_paths or list(mutations), + cwd=_gate_selection_cwd(state), ) return with_trace( "delegate_implementation", diff --git a/tests/test_gate_runner.py b/tests/test_gate_runner.py index 4384cad..b945d56 100644 --- a/tests/test_gate_runner.py +++ b/tests/test_gate_runner.py @@ -25,3 +25,36 @@ def test_docs_gate_reports_non_utf8_mutated_file(tmp_path) -> None: assert results[0]["returncode"] == 1 assert errors assert "UnicodeDecodeError" in errors[0]["stderr"] + + +def test_python_gate_uses_uv_dev_extra_when_available(tmp_path) -> None: + (tmp_path / "pyproject.toml").write_text( + "\n".join( + [ + "[project]", + 'name = "demo"', + 'version = "0.1.0"', + "[project.optional-dependencies]", + 'dev = ["ruff", "mypy", "pytest"]', + "", + ] + ), + encoding="utf-8", + ) + + commands = select_gate_commands_for_mutations(["src/app.py"], cwd=tmp_path) + + assert commands == [ + ["uv", "run", "--extra", "dev", "ruff", "check", "."], + ["uv", "run", "--extra", "dev", "mypy", "."], + ["uv", "run", "--extra", "dev", "python", "-m", "pytest", "-q"], + ] + + +def test_gate_reports_missing_tool_as_failure() -> None: + results, errors = run_gate_commands([["definitely-missing-hyrule-gate-tool"]]) + + assert results[0]["returncode"] == 127 + assert results[0]["status"] == "failed" + assert errors + assert "could not start" in errors[0]["message"] diff --git a/tests/test_phase20_agent_backend.py b/tests/test_phase20_agent_backend.py index e1b442e..b35af28 100644 --- a/tests/test_phase20_agent_backend.py +++ b/tests/test_phase20_agent_backend.py @@ -148,6 +148,42 @@ def test_subprocess_backend_command_assembly_and_refusals(tmp_path: Path) -> Non assert "acceptEdits" not in read_only_command +def test_pi_backend_defaults_to_json_mode_and_normalizes_usage_payload() -> None: + backend = PiBackend() + spec = TaskSpec( + change_id="PI_JSON", + change_class="app_feature", + risk_level="low", + request="measure this run", + allowed_paths={"hyrule-cloud": ("docs",)}, + ) + command = backend.build_command( + prompt=assemble_backend_prompt(spec, BackendConstraints(max_iterations=3)), + constraints=BackendConstraints(max_iterations=3), + ) + + assert command[:3] == ["pi", "--print", "--mode"] + assert command[3] == "json" + + parsed = backend._parse_harness_output( + json.dumps( + { + "numTurns": "3", + "cost": {"usd": "0.0425"}, + "usage": {"prompt_tokens": "1200", "completion_tokens": 240}, + "output": "completed the requested edit", + } + ) + ) + + assert parsed["num_turns"] == 3 + assert parsed["total_cost_usd"] == 0.0425 + assert parsed["usage"]["input_tokens"] == 1200 + assert parsed["usage"]["output_tokens"] == 240 + assert parsed["result"] == "completed the requested edit" + assert parsed["is_error"] is False + + def test_backend_selection_follows_tier_escalation(tmp_path: Path) -> None: policy_path = tmp_path / "model-policy.yml" policy_path.write_text( diff --git a/tests/test_phase21_task_specs.py b/tests/test_phase21_task_specs.py index d0d9fe2..5596ab6 100644 --- a/tests/test_phase21_task_specs.py +++ b/tests/test_phase21_task_specs.py @@ -19,7 +19,7 @@ from hyrule_engineering_loop.feature import build_feature_state from hyrule_engineering_loop.graph import build_graph from hyrule_engineering_loop.judgment import run_agentic_evaluation -from hyrule_engineering_loop.nodes import required_roles_for_state +from hyrule_engineering_loop.nodes import planner_node, required_roles_for_state from hyrule_engineering_loop.prompts import load_role_prompts from hyrule_engineering_loop.promotion import rollback_promotions from hyrule_engineering_loop.state import GraphState @@ -160,6 +160,29 @@ def test_planner_failure_routes_to_human_signoff(tmp_path: Path) -> None: assert "delegate_implementation" not in nodes +def test_planner_uses_issue_acceptance_section(tmp_path: Path) -> None: + state = _feature_state(tmp_path, "PLANNER_AC") + state["feature_request"] = "\n".join( + [ + "Improve the monitoring runbook.", + "", + "## Acceptance criteria", + "- [ ] Runbook states the alert owner.", + "1. Dashboard link is added to the evidence section.", + "", + "## Non-goals", + "- No runtime changes.", + ] + ) + + update = planner_node(state) + + assert update["task_spec"]["acceptance_criteria"] == [ + "Runbook states the alert owner.", + "Dashboard link is added to the evidence section.", + ] + + # --- AC2: consult + judgment recorded --------------------------------------- diff --git a/tests/test_phase24_daemon.py b/tests/test_phase24_daemon.py index 3bbf903..7618fd8 100644 --- a/tests/test_phase24_daemon.py +++ b/tests/test_phase24_daemon.py @@ -299,6 +299,43 @@ def test_daemon_cli_per_run_budget_flags() -> None: assert args.reliability_decision_author == ["trusted-governor"] +def test_daemon_issue_budget_label_widens_backend_budget(tmp_path: Path) -> None: + captured: dict[str, Any] = {} + repo = "AS215932/hyrule-cloud" + + def runner(**kwargs: Any) -> dict[str, Any]: + captured.update(kwargs) + return {"final_state": {}, "state_path": str(tmp_path / "state.json")} + + config = DaemonConfig( + repos=(repo,), + state_dir=tmp_path / "state", + output_root=tmp_path / "runs", + max_iterations_per_run=20, + max_wall_clock_minutes_per_run=45, + max_cost_usd_per_run=5.0, + ) + gh = FakeGh( + { + "issue list": _approved_issue_json( + 1, + repo=repo, + labels=["loop:approved", "loop:budget-xl"], + ), + "issue view": json.dumps({"body": "## Context\nfeature-class run\n"}), + } + ) + + daemon_once(config, client=gh, feature_runner=runner) + + assert captured["backend_budget"] == { + "max_iterations": 80, + "max_wall_clock_minutes": 180, + "max_cost_usd": 20.0, + "tier": "xl", + } + + def test_daemon_defaults_to_core_repos_and_low_and_slow_budget() -> None: config = DaemonConfig() assert config.repos == CORE_REPOS