diff --git a/actions/setup/js/handle_agent_failure.cjs b/actions/setup/js/handle_agent_failure.cjs index 048eb01c8e6..933f4125e49 100644 --- a/actions/setup/js/handle_agent_failure.cjs +++ b/actions/setup/js/handle_agent_failure.cjs @@ -64,6 +64,8 @@ const ENGINE_RATE_LIMIT_429_RE = /(?:\b429\b[\s\S]{0,120}(?:too many requests|rate[\s-]*limit)|\brate_limit_(?:error|exceeded)\b|capierror:\s*429|failed to get response from the ai model[\s\S]{0,120}\b429\b|exceeded your rate limit for utility models)/i; const ENGINE_MAX_RUNS_EXCEEDED_RE = /(?:\bmax_runs_exceeded\b|\bmaximum\s+llm\s+invocations\s+exceeded\b)/i; const ALLOWED_FILES_ERROR_RE = /^(?.*outside the allowed-files list) \((?.+?)\)\. (?Add the files to the allowed-files configuration field or remove them from the (?:patch|bundle)\.)$/; +const SANDBOX_NETWORK_INIT_FAILED_CODE = "SANDBOX_NETWORK_INIT_FAILED"; +const CLOUD_HYPERVISOR_GUEST_CONNECTIVITY_FAILURE_RE = /(?:SANDBOX_NETWORK_INIT_FAILED|Cloud Hypervisor guest connectivity probe failed)/i; /** * Parse action failure issue expiration from environment. @@ -2696,6 +2698,43 @@ function detectAWFStartupSignals(logContent, errorMessages = undefined) { return { isFirewallFailed, hasDNSFailure, hasDNSEAIAgain }; } +/** + * Detect Cloud Hypervisor guest networking preflight failures before engine startup. + * @param {string} logContent Full content of agent-stdio.log + * @returns {boolean} + */ +function hasSandboxNetworkInitFailureSignal(logContent) { + return CLOUD_HYPERVISOR_GUEST_CONNECTIVITY_FAILURE_RE.test(logContent); +} + +/** + * Render Cloud Hypervisor guest networking state lines from AWF probe output. + * @param {string[]} lines + * @returns {string[]} + */ +function extractSandboxNetworkStateLines(lines) { + return lines.filter(line => line.includes("guest network state:")).slice(-20); +} + +/** + * Build a dedicated context for Cloud Hypervisor guest network initialization failures. + * @param {string[]} lines + * @param {string} engineLabel + * @param {string[]} maskedValues + * @returns {string} + */ +function buildSandboxNetworkInitFailureContext(lines, engineLabel, maskedValues) { + let context = buildWarningAlertLine("Sandbox Network Init Failed", `\`${SANDBOX_NETWORK_INIT_FAILED_CODE}\`: Cloud Hypervisor guest networking did not initialize, so the${engineLabel} agent was never invoked.`) + "\n"; + const networkStateLines = extractSandboxNetworkStateLines(lines); + if (networkStateLines.length > 0) { + context += "**Guest network state:**\n```text\n"; + context += applyAddMaskRedaction(networkStateLines.join("\n"), maskedValues); + context += "\n```\n\n"; + } + context += "Check the preceding Cloud Hypervisor logs for the guest connectivity probe output and retry attempts.\n\n"; + return context; +} + /** * Detect whether the agent-stdio.log contains evidence of an AWF firewall startup failure. * Reads the log file from the path derived from GH_AW_AGENT_OUTPUT, falling back to the @@ -2808,6 +2847,11 @@ function buildEngineFailureContext(options = {}) { return ""; } + if (hasSandboxNetworkInitFailureSignal(logContent)) { + core.info("Detected Cloud Hypervisor guest networking preflight failure — using dedicated sandbox context message"); + return buildSandboxNetworkInitFailureContext(lines, engineLabel, maskedValues); + } + if (!suppressEngineRateLimit429 && (hasEngineRateLimit429Signal(logContent) || hasEngineRateLimit429InOTELMirror())) { core.info("Detected engine HTTP 429/rate-limit signal — using dedicated context message"); return buildEngineRateLimit429Context(engineLabel); diff --git a/actions/setup/js/handle_agent_failure.test.cjs b/actions/setup/js/handle_agent_failure.test.cjs index 1b3b7da71eb..6757cf7c772 100644 --- a/actions/setup/js/handle_agent_failure.test.cjs +++ b/actions/setup/js/handle_agent_failure.test.cjs @@ -2539,6 +2539,27 @@ describe("handle_agent_failure", () => { expect(result).toContain("> [!WARNING]"); }); + it("returns dedicated context for Cloud Hypervisor guest network init failures", () => { + process.env.GH_AW_ENGINE_ID = "claude"; + fs.writeFileSync( + stdioLogPath, + [ + "[WARN] [cloud-hypervisor] stage=guest-connectivity status=failed: Cloud Hypervisor guest connectivity probe failed with exit code 4", + "(stderr: Connection to 172.30.0.10 3128 port [tcp/*] succeeded!; guest network state: 1: lo: mtu 65536 qdisc noop state DOWN group default qlen 1000", + "ERR_CONFIG: Claude execution failed: no structured log entries were produced", + "", + ].join("\n") + ); + + const result = buildEngineFailureContext(); + expect(result).toContain("Sandbox Network Init Failed"); + expect(result).toContain("SANDBOX_NETWORK_INIT_FAILED"); + expect(result).toContain("guest network state:"); + expect(result).toContain("state DOWN"); + expect(result).not.toContain("Engine Failure"); + expect(result).not.toContain("Claude execution failed"); + }); + it("returns dedicated context for engine 429/rate-limit failures in stdio logs", () => { fs.writeFileSync(stdioLogPath, "Failed to get response from the AI model; retried 5 times. Last error: CAPIError: 429 429 Sorry, you've exceeded your rate limit for utility models.\n"); const result = buildEngineFailureContext(); diff --git a/pkg/workflow/awf_command_builder.go b/pkg/workflow/awf_command_builder.go index 18d408a3815..cef11a84374 100644 --- a/pkg/workflow/awf_command_builder.go +++ b/pkg/workflow/awf_command_builder.go @@ -14,6 +14,11 @@ import ( "github.com/github/gh-aw/pkg/workflow/compilerenv" ) +const ( + cloudHypervisorGuestConnectivityMaxAttempts = 2 + cloudHypervisorGuestConnectivityRetryBackoffSeconds = 3 +) + // BuildAWFCommand builds a complete AWF command with all arguments. // This consolidates the AWF command building logic that was duplicated across // Copilot, Claude, and Codex engines. @@ -64,6 +69,7 @@ func BuildAWFCommand(config AWFCommandConfig) string { awfArgs: awfArgs, shellWrappedCommand: shellWrappedCommand, logFile: config.LogFile, + isCloudHypervisor: isCloudHypervisor, }) awfHelpersLog.Print("Successfully built AWF command") @@ -268,6 +274,7 @@ type buildAWFCommandScriptInput struct { awfArgs []string shellWrappedCommand string logFile string + isCloudHypervisor bool } func buildAWFCommandScript(input buildAWFCommandScriptInput) string { @@ -289,20 +296,69 @@ func buildAWFCommandScript(input buildAWFCommandScriptInput) string { input.arcDindPrefixProbe, input.toolCacheMountProbe, awfShellcheckDirective, - fmt.Sprintf(`%s %s %s %s %s \ - -- %s 2>&1 | tee -a %s`, - input.awfCommand, - input.expandableArgs, - input.toolCacheMountRef, - input.arcDindDockerHostRef, - shellJoinArgs(input.awfArgs), - input.shellWrappedCommand, - shellEscapeArg(input.logFile), - ), + buildAWFInvocation(input), ) return strings.Join(lines, "\n") } +func buildAWFInvocation(input buildAWFCommandScriptInput) string { + awfInvocation := fmt.Sprintf(`%s %s %s %s %s \ + -- %s`, + input.awfCommand, + input.expandableArgs, + input.toolCacheMountRef, + input.arcDindDockerHostRef, + shellJoinArgs(input.awfArgs), + input.shellWrappedCommand, + ) + logFile := shellEscapeArg(input.logFile) + if !input.isCloudHypervisor { + return fmt.Sprintf("%s 2>&1 | tee -a %s", awfInvocation, logFile) + } + return fmt.Sprintf(`set -o pipefail +GH_AW_CLOUD_HYPERVISOR_MAX_ATTEMPTS=%d +GH_AW_CLOUD_HYPERVISOR_ATTEMPT=1 +GH_AW_CLOUD_HYPERVISOR_STATUS=1 +while true; do + GH_AW_CLOUD_HYPERVISOR_ATTEMPT_LOG="$(mktemp)" + echo "[INFO] [cloud-hypervisor] Starting guest connectivity preflight attempt ${GH_AW_CLOUD_HYPERVISOR_ATTEMPT}/${GH_AW_CLOUD_HYPERVISOR_MAX_ATTEMPTS}" + set +e + %s 2>&1 | tee "${GH_AW_CLOUD_HYPERVISOR_ATTEMPT_LOG}" | tee -a %s + GH_AW_CLOUD_HYPERVISOR_PIPE_STATUS=("${PIPESTATUS[@]}") + set -e + GH_AW_CLOUD_HYPERVISOR_STATUS="${GH_AW_CLOUD_HYPERVISOR_PIPE_STATUS[0]}" + if [ "${GH_AW_CLOUD_HYPERVISOR_STATUS}" -eq 0 ] && [ "${GH_AW_CLOUD_HYPERVISOR_PIPE_STATUS[1]}" -eq 0 ] && [ "${GH_AW_CLOUD_HYPERVISOR_PIPE_STATUS[2]}" -eq 0 ]; then + rm -f "${GH_AW_CLOUD_HYPERVISOR_ATTEMPT_LOG}" + break + fi + if grep -Fq "Cloud Hypervisor guest connectivity probe failed" "${GH_AW_CLOUD_HYPERVISOR_ATTEMPT_LOG}"; then + if [ "${GH_AW_CLOUD_HYPERVISOR_ATTEMPT}" -lt "${GH_AW_CLOUD_HYPERVISOR_MAX_ATTEMPTS}" ]; then + echo "::warning title=Cloud Hypervisor guest networking preflight retry::Cloud Hypervisor guest connectivity probe failed on attempt ${GH_AW_CLOUD_HYPERVISOR_ATTEMPT}/${GH_AW_CLOUD_HYPERVISOR_MAX_ATTEMPTS}; retrying after %d seconds." + rm -f "${GH_AW_CLOUD_HYPERVISOR_ATTEMPT_LOG}" + GH_AW_CLOUD_HYPERVISOR_ATTEMPT=$((GH_AW_CLOUD_HYPERVISOR_ATTEMPT + 1)) + sleep %d + continue + fi + echo "::error title=SANDBOX_NETWORK_INIT_FAILED::SANDBOX_NETWORK_INIT_FAILED: Cloud Hypervisor guest networking did not initialize; guest connectivity probe failed after ${GH_AW_CLOUD_HYPERVISOR_ATTEMPT} attempt(s)." + if grep -Fq "guest network state:" "${GH_AW_CLOUD_HYPERVISOR_ATTEMPT_LOG}"; then + echo "SANDBOX_NETWORK_INIT_FAILED guest network state:" + grep -F "guest network state:" "${GH_AW_CLOUD_HYPERVISOR_ATTEMPT_LOG}" | tail -n 20 + fi + else + if [ "${GH_AW_CLOUD_HYPERVISOR_STATUS}" -eq 0 ]; then + echo "::error title=Cloud Hypervisor log capture failed::Cloud Hypervisor AWF command succeeded, but log capture failed (tee statuses: ${GH_AW_CLOUD_HYPERVISOR_PIPE_STATUS[1]}, ${GH_AW_CLOUD_HYPERVISOR_PIPE_STATUS[2]})." + else + echo "[ERROR] [cloud-hypervisor] AWF command failed with exit code ${GH_AW_CLOUD_HYPERVISOR_STATUS}; no guest connectivity probe failure was detected." + fi + fi + rm -f "${GH_AW_CLOUD_HYPERVISOR_ATTEMPT_LOG}" + if [ "${GH_AW_CLOUD_HYPERVISOR_STATUS}" -eq 0 ]; then + GH_AW_CLOUD_HYPERVISOR_STATUS=1 + fi + exit "${GH_AW_CLOUD_HYPERVISOR_STATUS}" +done`, cloudHypervisorGuestConnectivityMaxAttempts, awfInvocation, logFile, cloudHypervisorGuestConnectivityRetryBackoffSeconds, cloudHypervisorGuestConnectivityRetryBackoffSeconds) +} + // BuildAWFArgs constructs common AWF arguments from configuration. // This extracts the shared AWF argument building logic from engine implementations. // diff --git a/pkg/workflow/cloud_hypervisor_test.go b/pkg/workflow/cloud_hypervisor_test.go index 4d7e84d91f7..354f6a0402f 100644 --- a/pkg/workflow/cloud_hypervisor_test.go +++ b/pkg/workflow/cloud_hypervisor_test.go @@ -121,12 +121,33 @@ func TestCloudHypervisorAWFCommandOmitsUnsupportedMountsAndTTY(t *testing.T) { command := BuildAWFCommand(config) assert.Contains(t, command, "sudo --preserve-env awf") assert.Contains(t, command, `--cloud-hypervisor-virtiofsd-sha256 "${GH_AW_CLOUD_HYPERVISOR_VIRTIOFSD_SHA256}"`) + assert.Contains(t, command, "GH_AW_CLOUD_HYPERVISOR_MAX_ATTEMPTS=2") + assert.Contains(t, command, "Cloud Hypervisor guest connectivity probe failed") + assert.Contains(t, command, "SANDBOX_NETWORK_INIT_FAILED") + assert.Contains(t, command, "guest network state:") assert.NotContains(t, command, "--mount") assert.NotContains(t, command, "--tty") assert.NotContains(t, command, "--legacy-security") assert.NotContains(t, command, "--enable-host-access") } +func TestDefaultAWFCommandDoesNotUseCloudHypervisorRetryWrapper(t *testing.T) { + config := AWFCommandConfig{ + EngineName: "claude", + WorkflowData: &WorkflowData{ + EngineConfig: &EngineConfig{ID: "claude"}, + NetworkPermissions: &NetworkPermissions{ + Firewall: &FirewallConfig{Enabled: true}, + }, + SandboxConfig: &SandboxConfig{Agent: &AgentSandboxConfig{ID: "awf", Runtime: AgentRuntimeDocker}}, + }, + } + + command := BuildAWFCommand(config) + assert.NotContains(t, command, "GH_AW_CLOUD_HYPERVISOR_MAX_ATTEMPTS") + assert.NotContains(t, command, "SANDBOX_NETWORK_INIT_FAILED") +} + func TestCloudHypervisorFirewallLogsUsePrivilegedMode(t *testing.T) { workflowData := &WorkflowData{ SandboxConfig: &SandboxConfig{Agent: &AgentSandboxConfig{