diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index 94586043..7a80e445 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -143,8 +143,13 @@ jobs: ) runs-on: ubuntu-latest permissions: + actions: read contents: read id-token: write + outputs: + r_package_peer_result: ${{ steps.r_package_peer.outputs.result }} + r_package_peer_conclusion: ${{ steps.r_package_peer.outputs.conclusion }} + r_package_peer_url: ${{ steps.r_package_peer.outputs.run_url }} env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true steps: @@ -272,6 +277,99 @@ jobs: git -C "$COVERAGE_SOURCE_WORKDIR" status --short tar -cf "$COVERAGE_SOURCE_ARCHIVE" -C "$COVERAGE_SOURCE_WORKDIR" . + - name: Await current-head R CMD check package evidence + id: r_package_peer + env: + GH_TOKEN: ${{ steps.coverage_read_app_token.outputs.token || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} + TARGET_REPOSITORY: ${{ needs.validate-pr-metadata.outputs.target_repository }} + PR_BASE_SHA: ${{ needs.validate-pr-metadata.outputs.base_sha }} + PR_HEAD_SHA: ${{ needs.validate-pr-metadata.outputs.head_sha }} + COVERAGE_SOURCE_WORKDIR: ${{ runner.temp }}/opencode-coverage-source + run: | + set -euo pipefail + + write_result() { + local result="$1" + local conclusion="${2:-}" + local run_url="${3:-}" + { + printf 'result=%s\n' "$result" + printf 'conclusion=%s\n' "$conclusion" + printf 'run_url=%s\n' "$run_url" + } >>"$GITHUB_OUTPUT" + } + + if [ ! -f "${COVERAGE_SOURCE_WORKDIR}/DESCRIPTION" ]; then + echo "No root R package DESCRIPTION exists; peer R CMD check evidence is not applicable." + write_result "not_applicable" + exit 0 + fi + + changed_paths="$(git -C "$COVERAGE_SOURCE_WORKDIR" diff --name-only "$PR_BASE_SHA" "$PR_HEAD_SHA")" + if ! rg -q '(^DESCRIPTION$|^NAMESPACE$|^R/.*[.]R$|^tests/testthat/.*[.]R$)' <<<"$changed_paths"; then + echo "The current head does not change the root R package or its testthat suite; peer R CMD check evidence is not applicable." + write_result "not_applicable" + exit 0 + fi + + mapfile -t workflow_paths < <( + rg -l \ + --glob '*.yml' \ + --glob '*.yaml' \ + 'r-lib/actions/check-r-package@' \ + "${COVERAGE_SOURCE_WORKDIR}/.github/workflows" 2>/dev/null \ + | while IFS= read -r path; do + printf '%s\n' "${path#${COVERAGE_SOURCE_WORKDIR}/}" + done + ) + if [ "${#workflow_paths[@]}" -eq 0 ]; then + echo "::error::Root R package changes require a repository R CMD check workflow using the pinned r-lib check action." + write_result "missing" + exit 0 + fi + + workflow_paths_json="$(printf '%s\n' "${workflow_paths[@]}" | jq -Rsc 'split("\n") | map(select(length > 0))')" + deadline=$((SECONDS + 5400)) + while [ "$SECONDS" -lt "$deadline" ]; do + runs_json="$( + gh api --method GET "repos/${TARGET_REPOSITORY}/actions/runs" \ + -f "head_sha=${PR_HEAD_SHA}" \ + -f event=pull_request \ + -F per_page=100 + )" + current_run="$( + jq -c \ + --arg head_sha "$PR_HEAD_SHA" \ + --argjson workflow_paths "$workflow_paths_json" \ + '[.workflow_runs[] + | select(.head_sha == $head_sha) + | select(.path as $path | $workflow_paths | index($path))] + | sort_by(.run_number, .run_attempt) + | last // empty' <<<"$runs_json" + )" + if [ -n "$current_run" ]; then + status="$(jq -r '.status // empty' <<<"$current_run")" + conclusion="$(jq -r '.conclusion // empty' <<<"$current_run")" + run_url="$(jq -r '.html_url // empty' <<<"$current_run")" + run_id="$(jq -r '.id // empty' <<<"$current_run")" + printf 'Current-head R CMD check run %s is %s/%s for %s.\n' "$run_id" "${status:-unknown}" "${conclusion:-pending}" "$PR_HEAD_SHA" + if [ "$status" = "completed" ]; then + if [ "$conclusion" = "success" ]; then + write_result "success" "$conclusion" "$run_url" + else + write_result "failure" "${conclusion:-unknown}" "$run_url" + fi + exit 0 + fi + else + printf 'Waiting for a current-head R CMD check run for %s.\n' "$PR_HEAD_SHA" + fi + sleep 30 + done + + echo "::error::Current-head R CMD check did not complete within 90 minutes. The package check may still be running; its result was not discarded or mislabeled as a code failure." + write_result "timed_out" + - name: Upload materialized pull request merge tree uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -385,6 +483,9 @@ jobs: PR_BASE_SHA: ${{ needs.validate-pr-metadata.outputs.base_sha }} PR_HEAD_SHA: ${{ needs.validate-pr-metadata.outputs.head_sha }} COVERAGE_SOURCE_WORKDIR: ${{ github.workspace }}/pr-head + R_PACKAGE_PEER_RESULT: ${{ needs.coverage-source-tree.outputs.r_package_peer_result }} + R_PACKAGE_PEER_CONCLUSION: ${{ needs.coverage-source-tree.outputs.r_package_peer_conclusion }} + R_PACKAGE_PEER_URL: ${{ needs.coverage-source-tree.outputs.r_package_peer_url }} # Dependency resolution may consume wheels/packages, but PR-defined # install/build hooks are never executed implicitly. UV_NO_BUILD: "1" @@ -996,8 +1097,34 @@ jobs: Rscript -e 'required <- c("covr", "testthat"); missing <- required[!vapply(required, requireNamespace, logical(1), quietly = TRUE)]; if (length(missing)) stop("signed distribution coverage packages unavailable: ", paste(missing, collapse = ", "))' if [ -f DESCRIPTION ]; then if [ -d tests/testthat ]; then - run_and_capture "R package testthat suite" \ - Rscript -e 'lib <- Sys.getenv("R_LIBS_USER"); .libPaths(c(lib, .libPaths())); if (!requireNamespace("testthat", quietly = TRUE)) { message("testthat unavailable in coverage runner; deferring to required peer R CMD check evidence."); quit(status = 0) }; testthat::test_dir("tests/testthat")' + case "${R_PACKAGE_PEER_RESULT:-not_applicable}" in + success) + append "### R package testthat suite" + append "" + append "- Result: PASS" + append "- Reason: the repository's current-head R CMD check completed successfully and provides package-aware installation plus testthat evidence." + if [ -n "${R_PACKAGE_PEER_URL:-}" ]; then + append "- Peer run: ${R_PACKAGE_PEER_URL}" + fi + append "" + ;; + missing|failure|timed_out) + append "### R package testthat suite" + append "" + append "- Result: FAIL" + append "- Reason: current-head package-aware R CMD check evidence is ${R_PACKAGE_PEER_RESULT} (conclusion: ${R_PACKAGE_PEER_CONCLUSION:-unavailable})." + if [ -n "${R_PACKAGE_PEER_URL:-}" ]; then + append "- Peer run: ${R_PACKAGE_PEER_URL}" + fi + append "- Fix: keep the pinned repository R CMD check workflow enabled and fix its exact installation or test failure before review approval." + append "" + failures=$((failures + 1)) + ;; + *) + run_and_capture "R package testthat suite" \ + Rscript -e 'lib <- Sys.getenv("R_LIBS_USER"); .libPaths(c(lib, .libPaths())); if (!requireNamespace("testthat", quietly = TRUE)) { message("testthat unavailable in coverage runner; no package-aware peer evidence exists."); quit(status = 1) }; testthat::test_dir("tests/testthat")' + ;; + esac else append "### R package testthat suite" append "" @@ -3368,7 +3495,7 @@ jobs: - name: Run OpenCode PR Review model pool id: opencode_review_model_pool if: needs.coverage-evidence.result == 'success' - timeout-minutes: 65 + timeout-minutes: 205 continue-on-error: true env: STRIX_GITHUB_MODELS_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }} @@ -3395,13 +3522,16 @@ jobs: # the SAME model 5x let a rate-limited/hung leader consume the whole # step, so the pool never reached a healthy fallback model. OPENCODE_MODEL_ATTEMPTS: "1" - # Keep stale providers from pinning required review jobs for hours. - # Adversarial validation needs enough room to read the evidence, but - # dynamic cadence and the outer watchdog still bound each current-head run. - OPENCODE_RUN_TIMEOUT_SECONDS: "600" - OPENCODE_EXPORT_TIMEOUT_SECONDS: "120" - OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600" - OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "2100" + # Preserve reviews that legitimately need tens of minutes to inspect a + # large repository. Changed-file count is not a reliable proxy for + # repository complexity, so every cadence class gets 90 minutes per + # model and a full-hour review is never discarded at the boundary. + # The 195-minute retry budget leaves room for a second substantial + # candidate while the outer 200-minute watchdog remains bounded. + OPENCODE_RUN_TIMEOUT_SECONDS: "5400" + OPENCODE_EXPORT_TIMEOUT_SECONDS: "180" + OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "11700" + OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "12000" # Keep cycling through the high-sensitivity candidate catalog until # the retry budget or step timeout is exhausted; a single invalid # cycle can be all provider formatting noise rather than review @@ -3410,16 +3540,16 @@ jobs: OPENCODE_DYNAMIC_REVIEW_CADENCE: "true" OPENCODE_SMALL_CHANGE_FILE_THRESHOLD: "3" OPENCODE_MEDIUM_CHANGE_FILE_THRESHOLD: "20" - OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "600" - OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "3600" - OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "600" - OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "3600" - OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "600" - OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "3600" - OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "600" - OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "3600" - OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS: "600" - OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS: "1800" + OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "5400" + OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "11700" + OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "5400" + OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "11700" + OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "5400" + OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "11700" + OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "5400" + OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "11700" + OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS: "5400" + OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS: "11700" OPENCODE_DYNAMIC_MAX_CYCLES_CAP: "0" # This installation currently reports a 4k request-body limit for # GitHub Models GPT-5 endpoints even though the public catalog is @@ -3429,8 +3559,8 @@ jobs: OPENCODE_DYNAMIC_MAX_CYCLES: "0" CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE: ${{ steps.central_review_process_fallback_scope.outputs.eligible || 'false' }} CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL: ${{ steps.central_review_process_fallback_scope.outputs.scope_label || 'unsupported' }} - OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "600" - OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "3600" + OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "5400" + OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "11700" OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_MAX_CYCLES: "1" OPENCODE_BACKOFF_INITIAL_SECONDS: "30" OPENCODE_BACKOFF_MAX_SECONDS: "30" diff --git a/scripts/ci/opencode_review_normalize_output.py b/scripts/ci/opencode_review_normalize_output.py index 4045d457..114963b7 100755 --- a/scripts/ci/opencode_review_normalize_output.py +++ b/scripts/ci/opencode_review_normalize_output.py @@ -653,6 +653,75 @@ def adversarial_probe_source_receipt_error( return "" +def repair_adversarial_probe_evidence_bindings(value: Any) -> dict[str, Any] | Any: + """Bind a verified structured probe location into its evidence text. + + Models can put the exact changed-file path and positive line in the + structured ``path``/``line`` fields while omitting the redundant + ``path:line`` citation from ``evidence``. Repair only that duplication when + the single receipt already matches the sealed current-head source line. + Every other validation failure remains untouched and therefore fails closed. + """ + if not isinstance(value, dict): + return value + validation = value.get("adversarial_validation") + if not isinstance(validation, dict): + return value + probes = validation.get("probes") + if not isinstance(probes, list): + return value + + changed_files = current_changed_files() + repaired_probes: list[Any] = [] + changed = False + for probe in probes: + if not isinstance(probe, dict): + repaired_probes.append(probe) + continue + path = probe.get("path") + line = probe.get("line") + evidence = probe.get("evidence") + if ( + not isinstance(path, str) + or path not in changed_files + or isinstance(line, bool) + or not isinstance(line, int) + or line <= 0 + or not isinstance(evidence, str) + or not evidence.strip() + or adversarial_probe_location_error(path, line) + ): + repaired_probes.append(probe) + continue + + receipts = SOURCE_LINE_RECEIPT_RE.findall(evidence) + if len(receipts) != 1 or adversarial_probe_source_receipt_error( + evidence, path, line + ): + repaired_probes.append(probe) + continue + rejection = adversarial_evidence_rejection_reason(evidence, path, line) + if rejection != "must cite the exact probe path and positive line": + repaired_probes.append(probe) + continue + + repaired_evidence = f"{path}:{line} {evidence}" + if adversarial_probe_source_receipt_error( + repaired_evidence, path, line + ) or adversarial_evidence_rejection_reason(repaired_evidence, path, line): + repaired_probes.append(probe) + continue + repaired_probes.append({**probe, "evidence": repaired_evidence}) + changed = True + + if not changed: + return value + return { + **value, + "adversarial_validation": {**validation, "probes": repaired_probes}, + } + + def adversarial_validation_error( value: Any, *, @@ -1267,6 +1336,7 @@ def reject(reason: str) -> None: return reject("APPROVE cannot contain findings") if result == "REQUEST_CHANGES" and not findings: return reject("REQUEST_CHANGES requires at least one finding") + value = repair_adversarial_probe_evidence_bindings(value) adversarial_error = adversarial_validation_error( value.get("adversarial_validation"), result=result, diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 85e122ab..d64fede7 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -92,8 +92,8 @@ env_integer_or_default() { cap_dynamic_cadence_for_queue() { local timeout_cap budget_cap cycle_cap previous_run_timeout previous_budget_seconds previous_max_cycles - timeout_cap="$(env_integer_or_default OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS 600)" - budget_cap="$(env_integer_or_default OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS 1800)" + timeout_cap="$(env_integer_or_default OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS 5400)" + budget_cap="$(env_integer_or_default OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS 11700)" cycle_cap="$(env_integer_or_default OPENCODE_DYNAMIC_MAX_CYCLES_CAP 0)" previous_run_timeout="$original_run_timeout" previous_budget_seconds="$budget_seconds" @@ -238,23 +238,33 @@ is_context_overflow_failure() { is_fatal_provider_failure() { local opencode_json_file="$1" + local opencode_stderr_file="${2:-/dev/null}" if is_context_overflow_failure "$opencode_json_file"; then return 0 fi - [ -s "$opencode_json_file" ] || return 1 - grep -Eiq 'budget limit|insufficient_quota' "$opencode_json_file" + grep -Eiq 'budget limit|insufficient_quota|statusCode"[[:space:]]*:[[:space:]]*5[0-9][0-9]|Model service is unavailable' \ + "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null } has_fatal_provider_error_event() { local opencode_json_file="$1" + local opencode_stderr_file="${2:-/dev/null}" - [ -s "$opencode_json_file" ] || return 1 # Only structured "type":"error" events count while the process is still # running: model prose or tool output quoting these signatures is # JSON-escaped inside event strings, so a healthy streaming run is never # killed for merely discussing context windows or quota errors. - awk 'tolower($0) ~ /"type"[[:space:]]*:[[:space:]]*"error"/ && tolower($0) ~ /contextoverflowerror|tokens_limit_reached|request body too large|context window|budget limit|insufficient_quota/ { found = 1; exit } END { exit !found }' "$opencode_json_file" + if [ -s "$opencode_json_file" ] && \ + awk 'tolower($0) ~ /"type"[[:space:]]*:[[:space:]]*"error"/ && tolower($0) ~ /contextoverflowerror|tokens_limit_reached|request body too large|context window|budget limit|insufficient_quota/ { found = 1; exit } END { exit !found }' "$opencode_json_file"; then + return 0 + fi + # OpenCode retries provider HTTP 5xx responses internally without emitting a + # JSON event. --print-logs writes the structured LLM error to the captured + # stderr artifact; fail over after the first service-unavailable response + # instead of paying for an hour-long exponential retry loop. + [ -s "$opencode_stderr_file" ] && \ + grep -Eiq 'service=llm.*statusCode"[[:space:]]*:[[:space:]]*5[0-9][0-9]|responseBody":"Model service is unavailable' "$opencode_stderr_file" } emit_sanitized_opencode_failure_detail() { @@ -272,7 +282,9 @@ emit_sanitized_opencode_failure_detail() { fi failure_class="unclassified" - if grep -Eiq 'ContextOverflowError|tokens_limit_reached|Request body too large|context window' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then + if grep -Eiq 'statusCode"[[:space:]]*:[[:space:]]*5[0-9][0-9]|Model service is unavailable' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then + failure_class="service-unavailable" + elif grep -Eiq 'ContextOverflowError|tokens_limit_reached|Request body too large|context window' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then failure_class="context-window" elif grep -Eiq 'budget limit|insufficient_quota|quota exceeded' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then failure_class="quota-or-budget" @@ -358,6 +370,57 @@ cap_model_run_timeout() { fi } +export_and_validate_session_output() { + local session_id="$1" + local model_candidate="$2" + local attempt="$3" + local attempts="$4" + local opencode_export_file="$5" + local candidate_output_file="$6" + local export_timeout_seconds="${OPENCODE_EXPORT_TIMEOUT_SECONDS:-180}" + + if ! timeout --kill-after=15s "${export_timeout_seconds}s" \ + env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ + opencode export "$session_id" --pure >"$opencode_export_file"; then + printf 'OpenCode %s attempt %s/%s session export did not complete within %ss.\n' "$model_candidate" "$attempt" "$attempts" "$export_timeout_seconds" + return 1 + fi + jq -r '.messages[] | select(.info.role == "assistant") | .parts[]? | select(.type == "text") | .text' "$opencode_export_file" >"$candidate_output_file" + if [ ! -s "$candidate_output_file" ]; then + printf 'OpenCode %s attempt %s/%s session export did not include assistant text.\n' "$model_candidate" "$attempt" "$attempts" + emit_rejected_opencode_artifact_metadata "assistant-empty-export" "$opencode_export_file" + return 1 + fi + if ! normalize_opencode_output "$candidate_output_file"; then + printf 'OpenCode %s attempt %s/%s output did not include a valid control conclusion.\n' "$model_candidate" "$attempt" "$attempts" + emit_rejected_opencode_artifact_metadata "invalid-control-output" "$candidate_output_file" + return 1 + fi + return 0 +} + +lookup_session_id_by_exact_title() { + local session_title="$1" + local session_list_file="$2" + local list_timeout_seconds="${OPENCODE_SESSION_LIST_TIMEOUT_SECONDS:-30}" + local session_id + + if ! timeout --kill-after=10s "${list_timeout_seconds}s" \ + env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ + opencode session list --pure --format json -n 100 >"$session_list_file"; then + printf 'OpenCode exact-title session lookup did not complete within %ss; timed-out work cannot be exported by fallback lookup.\n' "$list_timeout_seconds" >&2 + return 1 + fi + session_id="$(jq -r --arg title "$session_title" '[.[] | select(.title == $title) | .id] | last // empty' "$session_list_file" 2>/dev/null)" + if [ -z "$session_id" ] || [ "$session_id" = "null" ]; then + printf 'OpenCode exact-title session lookup found no session for the current run title.\n' >&2 + return 1 + fi + printf '%s\n' "$session_id" +} + run_one_model_attempt() { local model_candidate="$1" local attempt="$2" @@ -367,15 +430,17 @@ run_one_model_attempt() { local candidate_output_file="$6" local opencode_json_file="$7" local opencode_export_file="$8" - local run_timeout_seconds export_timeout_seconds opencode_status session_id opencode_stderr_file + local run_timeout_seconds opencode_status session_id opencode_stderr_file + local session_title session_list_file local opencode_pid fatal_poll_seconds - run_timeout_seconds="${OPENCODE_RUN_TIMEOUT_SECONDS:-600}" - export_timeout_seconds="${OPENCODE_EXPORT_TIMEOUT_SECONDS:-120}" + run_timeout_seconds="${OPENCODE_RUN_TIMEOUT_SECONDS:-5400}" fatal_poll_seconds="${OPENCODE_FATAL_ERROR_POLL_SECONDS:-5}" opencode_stderr_file="${opencode_json_file}.stderr" + session_list_file="${opencode_json_file}.sessions.json" + session_title="PR #${PR_NUMBER} OpenCode bounded review ${model_candidate} run ${RUN_ID:-local}/${RUN_ATTEMPT:-1} attempt ${attempt}/${attempts} nonce $$-${RANDOM}" - rm -f "$opencode_json_file" "$opencode_stderr_file" "$opencode_export_file" "$candidate_output_file" + rm -f "$opencode_json_file" "$opencode_stderr_file" "$opencode_export_file" "$candidate_output_file" "$session_list_file" set +e timeout --kill-after=30s "${run_timeout_seconds}s" \ env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ @@ -385,7 +450,9 @@ run_one_model_attempt() { --agent "$agent" \ --model "$model_candidate" \ --format json \ - --title "PR #${PR_NUMBER} OpenCode bounded review ${model_candidate} attempt ${attempt}/${attempts}" \ + --print-logs \ + --log-level ERROR \ + --title "$session_title" \ >"$opencode_json_file" 2>"$opencode_stderr_file" & opencode_pid=$! # Some providers (github-models ContextOverflowError) log a fatal error and @@ -393,7 +460,7 @@ run_one_model_attempt() { # log while opencode runs and kill the process early so the pool falls # through to the next candidate within seconds instead of minutes. while kill -0 "$opencode_pid" 2>/dev/null; do - if has_fatal_provider_error_event "$opencode_json_file"; then + if has_fatal_provider_error_event "$opencode_json_file" "$opencode_stderr_file"; then printf 'OpenCode %s attempt %s/%s logged a fatal provider error while still running; killing the hung process instead of waiting out the %ss run timeout.\n' \ "$model_candidate" "$attempt" "$attempts" "$run_timeout_seconds" kill "$opencode_pid" 2>/dev/null @@ -409,48 +476,51 @@ run_one_model_attempt() { wait "$opencode_pid" opencode_status=$? set -e + session_id="$(jq -r 'select(.type == "step_start") | .sessionID' "$opencode_json_file" 2>/dev/null | tail -n 1)" + if [ -z "$session_id" ] || [ "$session_id" = "null" ]; then + printf 'OpenCode %s attempt %s/%s JSON output did not expose a session id; attempting exact-title session-list recovery for the current run.\n' \ + "$model_candidate" "$attempt" "$attempts" + if session_id="$(lookup_session_id_by_exact_title "$session_title" "$session_list_file")"; then + printf 'OpenCode %s attempt %s/%s recovered the uniquely titled current-run session.\n' \ + "$model_candidate" "$attempt" "$attempts" + else + session_id="" + fi + fi if [ "$opencode_status" -ne 0 ]; then printf 'OpenCode %s attempt %s/%s failed with exit %s.\n' "$model_candidate" "$attempt" "$attempts" "$opencode_status" emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" - if [ "$opencode_status" -eq 124 ] || [ "$opencode_status" -eq 137 ]; then - printf 'OpenCode %s attempt %s/%s timed out after %ss; falling through within the remaining retry budget instead of blocking the org queue.\n' "$model_candidate" "$attempt" "$attempts" "$run_timeout_seconds" + if { [ "$opencode_status" -eq 124 ] || [ "$opencode_status" -eq 137 ] || [ "$opencode_status" -eq 143 ]; } && \ + ! is_fatal_provider_failure "$opencode_json_file" "$opencode_stderr_file"; then + printf 'OpenCode %s attempt %s/%s timed out after %ss; attempting to recover any assistant text already persisted in the uniquely titled current-run session before falling through.\n' \ + "$model_candidate" "$attempt" "$attempts" "$run_timeout_seconds" + if [ -n "$session_id" ] && [ "$session_id" != "null" ]; then + if export_and_validate_session_output "$session_id" "$model_candidate" "$attempt" "$attempts" "$opencode_export_file" "$candidate_output_file"; then + printf 'OpenCode %s attempt %s/%s recovered a valid control conclusion from the timed-out session; preserving paid review work.\n' \ + "$model_candidate" "$attempt" "$attempts" + return 0 + fi + printf 'OpenCode %s attempt %s/%s timed-out session export did not contain a valid recoverable conclusion; continuing within the remaining retry budget.\n' \ + "$model_candidate" "$attempt" "$attempts" + fi fi - if is_fatal_provider_failure "$opencode_json_file"; then - printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, or quota); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" + if is_fatal_provider_failure "$opencode_json_file" "$opencode_stderr_file"; then + printf 'OpenCode %s attempt %s/%s hit a fatal provider error (service unavailable, context window, token budget, or quota); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" return 2 fi return 1 fi - session_id="$(jq -r 'select(.type == "step_start") | .sessionID' "$opencode_json_file" | tail -n 1)" if [ -z "$session_id" ] || [ "$session_id" = "null" ]; then printf 'OpenCode %s attempt %s/%s JSON output did not include a session id.\n' "$model_candidate" "$attempt" "$attempts" emit_rejected_opencode_artifact_metadata "sessionless-json" "$opencode_json_file" - if is_fatal_provider_failure "$opencode_json_file"; then - printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, or quota); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" + if is_fatal_provider_failure "$opencode_json_file" "$opencode_stderr_file"; then + printf 'OpenCode %s attempt %s/%s hit a fatal provider error (service unavailable, context window, token budget, or quota); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" return 2 fi return 1 fi - if ! timeout --kill-after=15s "${export_timeout_seconds}s" \ - env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ - -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ - opencode export "$session_id" --pure >"$opencode_export_file"; then - printf 'OpenCode %s attempt %s/%s session export did not complete within %ss.\n' "$model_candidate" "$attempt" "$attempts" "$export_timeout_seconds" - return 1 - fi - jq -r '.messages[] | select(.info.role == "assistant") | .parts[]? | select(.type == "text") | .text' "$opencode_export_file" >"$candidate_output_file" - if [ ! -s "$candidate_output_file" ]; then - printf 'OpenCode %s attempt %s/%s session export did not include assistant text.\n' "$model_candidate" "$attempt" "$attempts" - emit_rejected_opencode_artifact_metadata "assistant-empty-export" "$opencode_export_file" - return 1 - fi - if ! normalize_opencode_output "$candidate_output_file"; then - printf 'OpenCode %s attempt %s/%s output did not include a valid control conclusion.\n' "$model_candidate" "$attempt" "$attempts" - emit_rejected_opencode_artifact_metadata "invalid-control-output" "$candidate_output_file" - return 1 - fi - return 0 + export_and_validate_session_output "$session_id" "$model_candidate" "$attempt" "$attempts" "$opencode_export_file" "$candidate_output_file" } main() { @@ -461,12 +531,12 @@ main() { local -a model_candidates attempts="${OPENCODE_MODEL_ATTEMPTS:-3}" - original_run_timeout="${OPENCODE_RUN_TIMEOUT_SECONDS:-600}" - budget_seconds="${OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-1500}" + original_run_timeout="${OPENCODE_RUN_TIMEOUT_SECONDS:-5400}" + budget_seconds="${OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-11700}" max_cycles="${OPENCODE_POOL_MAX_CYCLES:-0}" if [ "${CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE:-false}" = "true" ]; then - original_run_timeout="${OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS:-600}" - budget_seconds="${OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS:-3600}" + original_run_timeout="${OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS:-5400}" + budget_seconds="${OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS:-11700}" max_cycles="${OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_MAX_CYCLES:-1}" printf 'Central review-process evidence fallback eligible for scope "%s"; limiting OpenCode model pool to %ss per attempt, %ss total budget, and %s cycle(s) so provider delay is logged before the publish fallback evaluates current-head peer evidence.\n' \ "${CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL:-unsupported}" "$original_run_timeout" "$budget_seconds" "$max_cycles" @@ -475,22 +545,22 @@ main() { medium_file_threshold="$(env_integer_or_default OPENCODE_MEDIUM_CHANGE_FILE_THRESHOLD 20)" if changed_file_count="$(count_changed_files_for_cadence)"; then if [ "$changed_file_count" -le "$small_file_threshold" ]; then - original_run_timeout="$(env_integer_or_default OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS 900)" - budget_seconds="$(env_integer_or_default OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS 2100)" + original_run_timeout="$(env_integer_or_default OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS 5400)" + budget_seconds="$(env_integer_or_default OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS 11700)" elif [ "$changed_file_count" -le "$medium_file_threshold" ]; then - original_run_timeout="$(env_integer_or_default OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS 1800)" - budget_seconds="$(env_integer_or_default OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS 3900)" + original_run_timeout="$(env_integer_or_default OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS 5400)" + budget_seconds="$(env_integer_or_default OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS 11700)" else - original_run_timeout="$(env_integer_or_default OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS 3600)" - budget_seconds="$(env_integer_or_default OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS 7200)" + original_run_timeout="$(env_integer_or_default OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS 5400)" + budget_seconds="$(env_integer_or_default OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS 11700)" fi max_cycles="$(env_integer_or_default OPENCODE_DYNAMIC_MAX_CYCLES 0)" cap_dynamic_cadence_for_queue printf 'OpenCode dynamic review cadence selected %ss per attempt and %ss total budget for %s changed file(s); max-cycles=%s.\n' \ "$original_run_timeout" "$budget_seconds" "$changed_file_count" "$max_cycles" else - original_run_timeout="$(env_integer_or_default OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS 1800)" - budget_seconds="$(env_integer_or_default OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS 3900)" + original_run_timeout="$(env_integer_or_default OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS 5400)" + budget_seconds="$(env_integer_or_default OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS 11700)" max_cycles="$(env_integer_or_default OPENCODE_DYNAMIC_MAX_CYCLES 0)" cap_dynamic_cadence_for_queue printf 'OpenCode dynamic review cadence could not read OPENCODE_CHANGED_FILES_FILE; using %ss per attempt and %ss total budget; max-cycles=%s.\n' \ diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 3de19fa2..59468cb9 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -505,8 +505,13 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_not_contains "$workflow_file" 'install.packages(' "opencode R coverage never installs PR-selected mutable packages" assert_file_contains "$workflow_file" "libcurl4-openssl-dev libssl-dev libxml2-dev" "opencode R coverage installs system headers required by covr dependencies" assert_file_contains "$workflow_file" "r-cran-covr r-cran-testthat" "opencode R coverage uses signed distribution packages instead of mutable CRAN resolution" + assert_file_contains "$workflow_file" "Await current-head R CMD check package evidence" "opencode R package evidence waits for the package-aware current-head check" + assert_file_contains "$workflow_file" 'deadline=$((SECONDS + 5400))' "opencode R package evidence allows long repository checks to run for 90 minutes" + assert_file_contains "$workflow_file" 'select(.head_sha == $head_sha)' "opencode R package evidence is bound to the current PR head" + assert_file_contains "$workflow_file" 'select(.path as $path | $workflow_paths | index($path))' "opencode R package evidence accepts only the repository workflow path that declares the pinned check action" + assert_file_contains "$workflow_file" "current-head R CMD check completed successfully" "opencode R coverage reports the authoritative package-aware peer result" assert_file_contains "$workflow_file" "R package testthat suite" "opencode R package coverage requires package testthat evidence" - assert_file_contains "$workflow_file" "testthat unavailable in coverage runner; deferring to required peer R CMD check evidence." "opencode R package tests defer only when testthat cannot be installed in the coverage runner" + assert_file_contains "$workflow_file" "testthat unavailable in coverage runner; no package-aware peer evidence exists." "opencode R package tests fail closed when no peer package check exists" assert_file_contains "$workflow_file" "covr package_coverage unavailable after package tests; treating missing-line report as advisory." "opencode R package coverage does not block on covr installation reproduction after tests pass" assert_file_contains "$workflow_file" "signed distribution coverage packages unavailable" "opencode R coverage verifies distribution-provided covr/testthat are loadable" assert_file_contains "$workflow_file" "repository: ContextualWisdomLab/.github" "opencode required workflow checks out the central source repository" @@ -649,11 +654,15 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "billing details" "strix quick gate classifies provider quota starvation as infrastructure" assert_file_contains "$workflow_file" 'timeout-minutes: 150' "opencode review target releases stalled review runners within the bounded queue budget" assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode evidence preparation fails closed before it ties up the review queue" - assert_file_contains "$workflow_file" 'timeout-minutes: 65' "opencode model pool gives multiple candidates a bounded review window while capping stalled model attempts" + assert_file_contains "$workflow_file" 'timeout-minutes: 205' "opencode model pool gives two long-review candidates a bounded review window" assert_file_contains "$workflow_file" 'timeout-minutes: 34' "opencode fast approval publication is bounded around the dynamic image and package/GPU check wait" assert_file_contains "$workflow_file" 'continue-on-error: true' "opencode approval gate still runs after model-pool failure to publish a reason" - assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode primary review advances after a bounded stalled provider attempt" - assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600"' "opencode model pool exits before the step timeout so the approval gate can publish a reason" + assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' "opencode primary review preserves repositories that require more than one hour" + assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "11700"' "opencode model pool leaves enough budget for a second long-review candidate" + assert_file_contains "$workflow_file" 'OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "5400"' "opencode does not infer repository review cost from a small changed-file count" + assert_file_contains "$workflow_file" 'OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "5400"' "opencode medium changes preserve full-hour repository reviews" + assert_file_contains "$workflow_file" 'OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "5400"' "opencode large changes preserve full-hour repository reviews" + assert_file_contains "$workflow_file" 'OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "5400"' "opencode unknown changes fail safe with a long-review window" assert_file_contains "$workflow_file" 'OPENCODE_POOL_MAX_CYCLES: "0"' "opencode model pool keeps cycling until the bounded retry budget or step timeout is exhausted" assert_file_contains "$workflow_file" "needs.coverage-evidence.result == 'success'" "opencode model pool only runs after coverage evidence passed" assert_file_contains "$workflow_file" "id: opencode_review_model_pool" "opencode DeepSeek V3 fallback still runs after a primary model timeout or step failure when coverage evidence passed" @@ -801,11 +810,11 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OpenCode model pool has no configured model candidates." "opencode model pool fails fast when no candidates are configured" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OPENAI_API_KEY is not configured" "opencode model pool skips native OpenAI candidates when the org secret is absent" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "configured max cycle count" "opencode model pool exits before the job timeout after configured cycles" - assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-1500' "opencode model pool keeps a bounded default retry budget unless the workflow explicitly disables it" + assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-11700' "opencode model pool keeps a bounded 195-minute retry budget unless the workflow explicitly disables it" assert_file_not_contains "$workflow_file" "no model produced a valid review control block" "opencode model-failure path no longer documents a final exhausted state" assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode primary and fallback paths avoid multi-attempt stalls on one model" assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode catalog fallback tries each model once before moving on" - assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode catalog fallback advances after a bounded stalled provider attempt" + assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' "opencode catalog allows a 90-minute long-repository review" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OpenCode %s attempt %s/%s failed" "opencode catalog fallback records per-model retry failures" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "exponential backoff" "opencode model retry paths use exponential backoff instead of fixed sleeps" assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna github-models/openai/gpt-4.1 github-models/openai/gpt-5" "opencode review tries DeepSeek V3 before OpenAI fallbacks" @@ -1118,7 +1127,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" "collect_failed_check_evidence.sh" "opencode review workflow collects failed check logs and annotations" assert_file_contains "$workflow_file" 'HEAD_SHA: ${{ needs.validate-pr-metadata.outputs.head_sha }}' "opencode evidence step passes the live validated HEAD_SHA to failed-check evidence collection" assert_file_contains "$workflow_file" "FAILED_CHECK_EVIDENCE_ATTEMPTS" "opencode review workflow bounds waiting for peer check failures before model review" - assert_file_contains "$workflow_file" 'timeout-minutes: 65' "opencode model stage has a bounded multi-provider timeout" + assert_file_contains "$workflow_file" 'timeout-minutes: 205' "opencode model stage has a bounded multi-provider timeout" assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode evidence preparation has a bounded peer-check wait timeout" assert_file_contains "$workflow_file" 'FAILED_CHECK_EVIDENCE_ATTEMPTS: "6"' "opencode review workflow keeps pre-model peer-check waiting bounded for required workflow DX" assert_file_contains "$workflow_file" 'FAILED_CHECK_EVIDENCE_SLEEP_SECONDS: "5"' "opencode review workflow retries peer-check evidence without stalling the model stage for Strix-scale durations" diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 1ff1ff18..194e16c8 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -724,9 +724,10 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "skipping remaining attempts for this model" in model_pool_runner assert "using %ss run timeout with %ss retry budget remaining" in model_pool_runner assert ( - "timed out after %ss; falling through within the remaining retry budget" + "timed out after %ss; attempting to recover any assistant text already persisted" in model_pool_runner ) + assert "recovered a valid control conclusion from the timed-out session" in model_pool_runner assert "emit_sanitized_opencode_failure_detail" in model_pool_runner assert "OpenCode provider failure metadata" in model_pool_runner assert "provider-controlled content suppressed" in model_pool_runner @@ -797,11 +798,11 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE" in workflow assert "CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL" in workflow assert ( - 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "600"' + 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "5400"' in workflow ) assert ( - 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "3600"' + 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "11700"' in workflow ) assert 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_MAX_CYCLES: "1"' in workflow @@ -873,15 +874,15 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert re.search(r"opencode-review-target:[\s\S]*?timeout-minutes: 150", workflow) assert "timeout-minutes: 12" in workflow assert re.search( - r"Run OpenCode PR Review model pool[\s\S]{0,240}timeout-minutes: 65", workflow + r"Run OpenCode PR Review model pool[\s\S]{0,240}timeout-minutes: 205", workflow ) - assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' in workflow - assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "2100"' in workflow + assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' in workflow + assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "12000"' in workflow assert ( 'timeout --kill-after=30s "${OPENCODE_POOL_STEP_TIMEOUT_SECONDS:-3600}s"' in workflow @@ -924,26 +925,26 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): 'github-models/deepseek/deepseek-r1"' ) in workflow assert 'OPENCODE_MODEL_ATTEMPTS: "1"' in workflow - assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' in workflow - assert 'OPENCODE_EXPORT_TIMEOUT_SECONDS: "120"' in workflow - assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "2100"' in workflow + assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' in workflow + assert 'OPENCODE_EXPORT_TIMEOUT_SECONDS: "180"' in workflow + assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "12000"' in workflow assert 'OPENCODE_POOL_MAX_CYCLES: "0"' in workflow assert 'OPENCODE_DYNAMIC_REVIEW_CADENCE: "true"' in workflow assert ( "OPENCODE_CHANGED_FILES_FILE: ${{ runner.temp }}/opencode-changed-files.txt" in workflow ) - assert 'OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "600"' in workflow - assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "600"' in workflow - assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "600"' in workflow - assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "600"' in workflow - assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS: "600"' in workflow - assert 'OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS: "1800"' in workflow + assert 'OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "5400"' in workflow + assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "5400"' in workflow + assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "5400"' in workflow + assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "5400"' in workflow + assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS: "5400"' in workflow + assert 'OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS: "11700"' in workflow assert 'OPENCODE_DYNAMIC_MAX_CYCLES_CAP: "0"' in workflow assert 'OPENCODE_GITHUB_GPT5_RUN_TIMEOUT_SECONDS: "45"' in workflow assert 'OPENCODE_DYNAMIC_MAX_CYCLES: "0"' in workflow @@ -1018,7 +1019,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert ( "OpenCode model pool has no configured model candidates." in model_pool_runner ) - assert "OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-1500" in model_pool_runner + assert "OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-11700" in model_pool_runner assert ( "completed a full model-candidate cycle without a valid control conclusion" in model_pool_runner diff --git a/tests/test_opencode_existing_approval_gate.py b/tests/test_opencode_existing_approval_gate.py index 7602b2a1..1ea92777 100644 --- a/tests/test_opencode_existing_approval_gate.py +++ b/tests/test_opencode_existing_approval_gate.py @@ -35,6 +35,8 @@ def test_standalone_import_path_loads_both_top_level_helpers(monkeypatch): @pytest.fixture(autouse=True) def trusted_adversarial_artifacts(tmp_path, monkeypatch): """Provide sealed current-head source and changed-file evidence to the gate.""" + opencode_review_normalize_output.current_changed_files.cache_clear() + opencode_review_normalize_output.trusted_execution_receipts.cache_clear() runner_temp = tmp_path / "runner-temp" source_root = tmp_path / "source" source_path = source_root / ".github" / "workflows" / "opencode-review.yml" @@ -68,6 +70,9 @@ def trusted_adversarial_artifacts(tmp_path, monkeypatch): "OPENCODE_ARTIFACT_MANIFEST_SHA256", hashlib.sha256(manifest.read_bytes()).hexdigest(), ) + yield + opencode_review_normalize_output.current_changed_files.cache_clear() + opencode_review_normalize_output.trusted_execution_receipts.cache_clear() def valid_body(head: str = HEAD) -> str: diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 71e96add..a53cc5a6 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -153,12 +153,22 @@ def run_failed_model( fake_opencode.write_text( "#!/usr/bin/env bash\n" 'if [ "${1:-}" = run ]; then\n' + ' previous=""\n' + ' for argument in "$@"; do\n' + ' if [ "$previous" = "--title" ] && [ -n "${FAKE_OPENCODE_TITLE_FILE:-}" ]; then printf \'%s\n\' "$argument" > "$FAKE_OPENCODE_TITLE_FILE"; fi\n' + ' previous="$argument"\n' + ' done\n' ' [ -z "${FAKE_OPENCODE_PROMPT_CAPTURE:-}" ] || printf \'%s\\n\' "$2" > "$FAKE_OPENCODE_PROMPT_CAPTURE"\n' ' [ -z "${FAKE_OPENCODE_JSON:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_JSON"\n' ' [ -z "${FAKE_OPENCODE_STDERR:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_STDERR" >&2\n' ' sleep "${FAKE_OPENCODE_HANG_SECONDS:-0}"\n' ' exit "${FAKE_OPENCODE_RUN_EXIT:-1}"\n' "fi\n" + 'if [ "${1:-}" = session ] && [ "${2:-}" = list ]; then\n' + ' title="$(cat "${FAKE_OPENCODE_TITLE_FILE:-/dev/null}" 2>/dev/null || true)"\n' + ' if [ -n "${FAKE_OPENCODE_SESSION_ID:-}" ] && [ -n "$title" ]; then printf \'[{"id":"%s","title":"%s"}]\n\' "$FAKE_OPENCODE_SESSION_ID" "$title"; else printf \'[]\n\'; fi\n' + ' exit "${FAKE_OPENCODE_SESSION_LIST_EXIT:-0}"\n' + "fi\n" 'if [ "${1:-}" = export ]; then\n' ' [ -z "${FAKE_OPENCODE_EXPORT:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_EXPORT"\n' ' exit "${FAKE_OPENCODE_EXPORT_EXIT:-0}"\n' @@ -179,6 +189,7 @@ def run_failed_model( if prompt_capture else "", "FAKE_OPENCODE_STDERR": stderr_line, + "FAKE_OPENCODE_TITLE_FILE": bash_path(tmp_path / "opencode-title.txt"), "GITHUB_OUTPUT": bash_path(github_output), "GITHUB_WORKSPACE": bash_path(ROOT), "HEAD_SHA": "1" * 40, @@ -561,6 +572,30 @@ def test_fatal_provider_error_kills_hung_opencode_run_early( assert elapsed < 25 +def test_service_unavailable_log_kills_internal_retry_loop_early( + tmp_path: Path, +) -> None: + """A captured provider HTTP 5xx fails over instead of retrying for an hour.""" + result = run_failed_model( + tmp_path, + stderr_line=( + 'ERROR service=llm error={"statusCode":500,' + '"responseBody":"Model service is unavailable."}' + ), + extra_env={ + "FAKE_OPENCODE_HANG_SECONDS": "120", + "OPENCODE_RUN_TIMEOUT_SECONDS": "5400", + "OPENCODE_TOTAL_RETRY_BUDGET_SECONDS": "11700", + }, + model_candidates="github-models/deepseek/deepseek-v3-0324", + ) + + assert result.returncode == 1 + assert "logged a fatal provider error while still running" in result.stdout + assert "class=service-unavailable" in result.stdout + assert "skipping remaining attempts for this model" in result.stdout + + def test_model_text_quoting_error_signatures_does_not_kill_run(tmp_path: Path) -> None: """Model prose mentioning fatal signatures never kills a healthy streaming run.""" result = run_failed_model( @@ -576,6 +611,79 @@ def test_model_text_quoting_error_signatures_does_not_kill_run(tmp_path: Path) - assert "logged a fatal provider error while still running" not in result.stdout +def test_timed_out_session_exports_persisted_assistant_text_before_fallback( + tmp_path: Path, +) -> None: + """A paid model timeout exports persisted text instead of discarding it unseen.""" + result = run_failed_model( + tmp_path, + json_line='{"type":"step_start","sessionID":"paid-timeout-session"}', + extra_env={ + "FAKE_OPENCODE_HANG_SECONDS": "5", + "FAKE_OPENCODE_EXPORT": json.dumps( + { + "messages": [ + { + "info": {"role": "assistant"}, + "parts": [ + { + "type": "text", + "text": "persisted but incomplete review", + } + ], + } + ] + } + ), + "OPENCODE_RUN_TIMEOUT_SECONDS": "1", + "OPENCODE_TOTAL_RETRY_BUDGET_SECONDS": "3", + }, + model_candidates="github-models/deepseek/deepseek-v3-0324", + ) + + assert result.returncode == 1 + assert "attempting to recover any assistant text already persisted" in result.stdout + assert "kind=invalid-control-output" in result.stdout + assert ( + "timed-out session export did not contain a valid recoverable conclusion" + in result.stdout + ) + + +def test_timed_out_session_recovers_id_from_exact_current_run_title( + tmp_path: Path, +) -> None: + """Zero-byte JSON still locates and exports the uniquely titled paid session.""" + result = run_failed_model( + tmp_path, + extra_env={ + "FAKE_OPENCODE_HANG_SECONDS": "5", + "FAKE_OPENCODE_SESSION_ID": "session-list-paid-timeout", + "FAKE_OPENCODE_EXPORT": json.dumps( + { + "messages": [ + { + "info": {"role": "assistant"}, + "parts": [ + {"type": "text", "text": "persisted incomplete review"} + ], + } + ] + } + ), + "OPENCODE_RUN_TIMEOUT_SECONDS": "1", + "OPENCODE_TOTAL_RETRY_BUDGET_SECONDS": "3", + }, + model_candidates="github-models/deepseek/deepseek-v3-0324", + ) + + assert result.returncode == 1 + assert "attempting exact-title session-list recovery" in result.stdout + assert "recovered the uniquely titled current-run session" in result.stdout + assert "session-list-paid-timeout" not in result.stdout + assert "kind=invalid-control-output" in result.stdout + + def test_dynamic_review_cadence_uses_small_change_timeout(tmp_path: Path) -> None: """Small PRs fail through hung/unavailable providers quickly with a visible budget reason.""" result = run_failed_model( @@ -627,11 +735,11 @@ def test_dynamic_review_cadence_caps_large_change_queue_budget(tmp_path: Path) - assert result.returncode == 1 assert ( - "OpenCode dynamic review cadence queue cap applied: per-attempt 3600s -> 600s, " + "OpenCode dynamic review cadence queue cap applied: per-attempt 3600s -> 3600s, " "total budget 7200s -> 1s, max-cycles 0 -> 0" ) in result.stdout assert ( - "OpenCode dynamic review cadence selected 600s per attempt and 1s total budget " + "OpenCode dynamic review cadence selected 3600s per attempt and 1s total budget " "for 21 changed file(s); max-cycles=0." ) in result.stdout assert "OpenCode model pool reached configured max cycle count" not in result.stdout diff --git a/tests/test_opencode_review_normalize_output.py b/tests/test_opencode_review_normalize_output.py index 590fb3e5..40d709a0 100644 --- a/tests/test_opencode_review_normalize_output.py +++ b/tests/test_opencode_review_normalize_output.py @@ -242,6 +242,108 @@ def fail_target_read(path): ) +def test_valid_control_repairs_only_redundant_verified_probe_locations( + tmp_path, monkeypatch +): + """A sealed receipt may restore path:line text omitted by the model.""" + require_adversarial_validation(tmp_path, monkeypatch, "scripts/ci/example.py") + value = control(adversarial_validation=adversarial_validation()) + for probe in value["adversarial_validation"]["probes"]: + probe["evidence"] = ( + "Regression command rejected malformed input with exit code 1; " + + source_line_receipt(f"line {probe['line']}") + ) + + normalized = norm.valid_control( + value, + expected_head_sha="head", + expected_run_id="run", + expected_run_attempt="attempt", + ) + + assert normalized is not None + assert all( + probe["evidence"].startswith(f"{probe['path']}:{probe['line']} ") + for probe in normalized["adversarial_validation"]["probes"] + ) + + +@pytest.mark.parametrize( + "evidence", + [ + "Regression command rejected malformed input with exit code 1; " + + "source-line-sha256=" + + "0" * 64, + "Regression command rejected malformed input with exit code 1; {receipt} {receipt}", + "Source inspection properly handles all cases; {receipt}", + "No command was run; {receipt}", + ], +) +def test_probe_binding_repair_keeps_unverified_evidence_fail_closed( + tmp_path, monkeypatch, evidence +): + """Mismatched, duplicate, circular, or unobserved evidence is not repaired.""" + require_adversarial_validation(tmp_path, monkeypatch, "scripts/ci/example.py") + receipt = source_line_receipt("line 7") + probe_evidence = evidence.format(receipt=receipt) + value = control( + adversarial_validation=adversarial_validation(outcomes=("falsified",)) + ) + value["adversarial_validation"]["probes"][0]["evidence"] = probe_evidence + + repaired = norm.repair_adversarial_probe_evidence_bindings(value) + + assert repaired is value + + +def test_probe_binding_repair_rejects_malformed_structures(tmp_path, monkeypatch): + """Unstructured probes and invalid structured locations are never rewritten.""" + require_adversarial_validation(tmp_path, monkeypatch, "scripts/ci/example.py") + assert norm.repair_adversarial_probe_evidence_bindings(None) is None + + malformed = {"adversarial_validation": {"probes": "not-a-list"}} + assert norm.repair_adversarial_probe_evidence_bindings(malformed) is malformed + + mixed = { + "adversarial_validation": { + "probes": [ + "not-an-object", + { + "path": "scripts/ci/example.py", + "line": 0, + "evidence": source_line_receipt("line 7"), + }, + ] + } + } + assert norm.repair_adversarial_probe_evidence_bindings(mixed) is mixed + + +def test_probe_binding_repair_rechecks_the_completed_binding( + tmp_path, monkeypatch +): + """A repaired value is discarded if the final independent check rejects it.""" + require_adversarial_validation(tmp_path, monkeypatch, "scripts/ci/example.py") + value = control(adversarial_validation=adversarial_validation()) + probe = value["adversarial_validation"]["probes"][0] + probe["evidence"] = ( + "Regression command rejected malformed input with exit code 1; " + + source_line_receipt(f"line {probe['line']}") + ) + original_check = norm.adversarial_evidence_rejection_reason + + def reject_completed_binding(evidence, path, line): + if evidence.startswith(f"{path}:{line} "): + return "simulated final binding rejection" + return original_check(evidence, path, line) + + monkeypatch.setattr( + norm, "adversarial_evidence_rejection_reason", reject_completed_binding + ) + + assert norm.repair_adversarial_probe_evidence_bindings(value) is value + + def test_adversarial_validation_requires_two_falsified_material_probes( tmp_path, monkeypatch ):