From d6c435ad7cf9250d47a7a7f9fb861988ac676c70 Mon Sep 17 00:00:00 2001 From: openhands Date: Fri, 14 Aug 2026 13:39:19 +0000 Subject: [PATCH] fix(ci): stabilize flaky up-docker-wsl on windows-latest The up-docker-wsl job was intermittently failing with the container exiting (exit code 0) right after start, followed by "[inject] EOF" and 3 failed restart attempts. The rerun of the same job with identical code passed, confirming an environmental flake on windows-latest + Podman/WSL. Root cause: podman `machine start` returns before the WSL VM and its API socket are fully ready. Containers created in that window get SIGTERM'd mid-startup; the daemon (PID 1) catches SIGTERM via signal.NotifyContext and exits 0, which cascades into the inject EOF and the restart loop. This is the Windows analogue of the Linux Podman-machine flakiness PR #1039 addresses (which is gated runtime.GOOS == linux). Changes: * .github/workflows/pr-ci.yml: mirror the Linux rootful setup for the Windows Podman step - retry machine init/start (init is not idempotent, so only on the first attempt), gate the e2e suite behind a `podman info` readiness probe, and run a throwaway preflight container before the suite starts. * pkg/devcontainer/single.go: DefaultEntrypoint now execs the daemon from $DEVSY_AGENT_PATH (falling back to /usr/local/bin/devsy). This wires in LocalDockerDelivery's pre-start volume binary, so the daemon arms immediately from the volume instead of waiting for post-start shell injection to place /usr/local/bin/devsy - shrinking the startup race window. DEVSY_AGENT_PATH was previously set but had zero readers. * pkg/config/env.go: add the EnvAgentPath constant for the above. * pkg/agent/delivery/local_docker.go: use the new constant. * pkg/driver/docker/lifecycle.go: surface restart-attempt failures at warn instead of debug so the exit code + tail logs are visible without --debug (an exit code 0 here typically means PID 1 caught a SIGTERM). Co-authored-by: openhands --- .github/workflows/pr-ci.yml | 56 ++++++++++++++++++++++++++++-- pkg/agent/delivery/local_docker.go | 2 +- pkg/config/env.go | 7 ++++ pkg/devcontainer/single.go | 5 +-- pkg/driver/docker/lifecycle.go | 7 +++- 5 files changed, 70 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 7c66c1eca..7baad7e23 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -571,9 +571,59 @@ jobs: echo "$installDir" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8 wsl --set-default-version 2 - & "$installDir\podman.exe" machine init - & "$installDir\podman.exe" machine set --rootful - & "$installDir\podman.exe" machine start + + # podman machine start can return before the WSL VM and its API socket + # are fully ready, leaving a window where containers created immediately + # afterwards get SIGTERM'd mid-startup (the flaky "container exited + # (exit code 0)" + "[inject] EOF" cascade). Retry init/start a few + # times, then gate the e2e suite behind a readiness probe + a throwaway + # preflight container, mirroring the Linux rootful setup below. + $busybox = "busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d" + $machineReady = $false + for ($attempt = 1; $attempt -le 3; $attempt++) { + if ($attempt -gt 1) { + Write-Host "--- retrying podman machine start (attempt $attempt/3) ---" + & "$installDir\podman.exe" machine stop 2>$null + Start-Sleep -Seconds 3 + } else { + # machine init is not idempotent (errors if the machine already + # exists), so only create it on the first attempt; later attempts + # reuse the existing machine and just restart it. + & "$installDir\podman.exe" machine init + & "$installDir\podman.exe" machine set --rootful + } + & "$installDir\podman.exe" machine start + if ($LASTEXITCODE -ne 0) { + Write-Host "podman machine start exited $LASTEXITCODE on attempt $attempt" + continue + } + + $ready = $false + $deadline = (Get-Date).AddSeconds(60) + while ((Get-Date) -lt $deadline) { + & "$installDir\podman.exe" info *> $null + if ($LASTEXITCODE -eq 0) { $ready = $true; break } + Start-Sleep -Seconds 2 + } + if (-not $ready) { + Write-Host "::warning::podman info did not succeed within 60s on attempt $attempt" + & "$installDir\podman.exe" machine list + continue + } + + Write-Host "Running podman runtime preflight..." + & "$installDir\podman.exe" run --rm $busybox echo "podman runtime preflight OK" + if ($LASTEXITCODE -eq 0) { + $machineReady = $true + break + } + Write-Host "podman preflight container failed (exit $LASTEXITCODE) on attempt $attempt" + } + if (-not $machineReady) { + & "$installDir\podman.exe" machine list + & "$installDir\podman.exe" info + throw "podman machine did not become ready after 3 attempts" + } - name: cache kind.exe (Windows) if: matrix.install-kind == true && runner.os == 'Windows' && (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') diff --git a/pkg/agent/delivery/local_docker.go b/pkg/agent/delivery/local_docker.go index 24951fb6b..c5f635b26 100644 --- a/pkg/agent/delivery/local_docker.go +++ b/pkg/agent/delivery/local_docker.go @@ -66,7 +66,7 @@ func (d *LocalDockerDelivery) DeliverPreStart(ctx context.Context, opts PreStart if opts.RunOptions.Env == nil { opts.RunOptions.Env = make(map[string]string) } - opts.RunOptions.Env["DEVSY_AGENT_PATH"] = volumeMountPath + "/" + binaryName() + opts.RunOptions.Env[pkgconfig.EnvAgentPath] = volumeMountPath + "/" + binaryName() return nil } diff --git a/pkg/config/env.go b/pkg/config/env.go index 48ab7ef75..28ea82caf 100644 --- a/pkg/config/env.go +++ b/pkg/config/env.go @@ -60,6 +60,13 @@ const ( // EnvAgentPreferDownload forces agent binary download even if a local copy exists. EnvAgentPreferDownload = "DEVSY_AGENT_PREFER_DOWNLOAD" + // EnvAgentPath is set by pre-start delivery strategies (e.g. + // LocalDockerDelivery) to the in-container path of the volume-mounted agent + // binary. The container entrypoint honors it to exec the daemon directly + // from the delivered binary, decoupling daemon startup from post-start + // shell injection. + EnvAgentPath = "DEVSY_AGENT_PATH" + // EnvOS is set to the host operating system (runtime.GOOS). EnvOS = "DEVSY_OS" diff --git a/pkg/devcontainer/single.go b/pkg/devcontainer/single.go index ce1767bde..c9d5239cc 100644 --- a/pkg/devcontainer/single.go +++ b/pkg/devcontainer/single.go @@ -31,11 +31,12 @@ const ( RemoteContainersExtraEnvVar = "REMOTE_CONTAINERS" DefaultEntrypoint = ` -while ! command -v /usr/local/bin/devsy >/dev/null 2>&1; do +DEVSYPATH="${DEVSY_AGENT_PATH:-/usr/local/bin/devsy}" +while ! command -v "$DEVSYPATH" >/dev/null 2>&1; do echo "waiting for devsy agent to be available" sleep 1 done -exec /usr/local/bin/devsy internal agent container daemon +exec "$DEVSYPATH" internal agent container daemon ` ) diff --git a/pkg/driver/docker/lifecycle.go b/pkg/driver/docker/lifecycle.go index deafa4985..8968031de 100644 --- a/pkg/driver/docker/lifecycle.go +++ b/pkg/driver/docker/lifecycle.go @@ -92,7 +92,12 @@ func (d *dockerDriver) ensureContainerRunning( return err } lastErr = err - log.Debugf("container %s restart attempt %d failed: %v", container.ID, attempt, err) + // Surface restart failures at warn (not debug) so the container's exit + // code and tail logs are visible without --debug. An exit code 0 here + // usually means PID 1 caught a SIGTERM mid-startup (e.g. an unstable + // Podman/WSL machine), which is otherwise indistinguishable from a + // clean shutdown in the debug output. + log.Warnf("container %s restart attempt %d failed: %v", container.ID, attempt, err) } return fmt.Errorf(