diff --git a/.github/actions/ccache-setup/action.yml b/.github/actions/ccache-setup/action.yml index f68734209fa..31cea8d4686 100644 --- a/.github/actions/ccache-setup/action.yml +++ b/.github/actions/ccache-setup/action.yml @@ -42,9 +42,38 @@ runs: if command -v ccache >/dev/null 2>&1; then echo "ccache already installed: $(ccache --version | head -1)" elif [ "${{ runner.os }}" = "Linux" ]; then - sudo apt-get update -q - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ - --no-install-recommends ccache + export DEBIAN_FRONTEND=noninteractive + # install-apt-deps stages the WHOLE ghcr bundle into + # /var/cache/apt/archives, and ccache is in the -minimal/-full + # lists, so in a job that ran it first the .deb is already on disk. + # Take it offline (--no-download): no apt-get update, nothing to + # stall on. Every other path here reaches the mirror, which is what + # used to hang these jobs for 10-40 min after the bundle had + # already installed cleanly. + sudo dpkg --configure -a >/dev/null 2>&1 || true + if sudo apt-get install -y --no-install-recommends \ + --no-download ccache; then + echo "ccache installed offline from the staged .deb bundle" + else + # Same defence in depth as install-apt-deps: Acquire timeouts drop + # a stalled connection, `timeout` hard-kills a wedged apt-get, and + # only then does the retry loop get a non-zero exit to act on. + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30) + ok="" + for i in 1 2; do + sudo dpkg --configure -a >/dev/null 2>&1 || true + if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \ + sudo timeout -k 10 180 apt-get "${APT_OPTS[@]}" install -y \ + --no-install-recommends ccache; then + ok=1 + break + fi + echo "::warning::ccache apt install failed (attempt $i/2)" + sleep 5 + done + [ -n "$ok" ] || { echo "::error::could not install ccache"; exit 1; } + fi elif [ "${{ runner.os }}" = "macOS" ]; then brew install ccache else diff --git a/.github/actions/install-apt-deps/action.yml b/.github/actions/install-apt-deps/action.yml index b8cb68b4e4a..f61e81febf3 100644 --- a/.github/actions/install-apt-deps/action.yml +++ b/.github/actions/install-apt-deps/action.yml @@ -5,9 +5,12 @@ inputs: description: 'Space-separated list of apt packages to install' required: true retries: - description: 'Number of retry attempts' + description: > + Number of retry attempts. Keep attempts x (60s update + 300s install) + within the calling job's timeout-minutes, or the last attempt is cut + off before it can report. required: false - default: '3' + default: '2' retry-delay: description: 'Initial delay between retries (seconds, doubles each attempt)' required: false @@ -50,7 +53,7 @@ runs: # PRs read the public upstream image too rather than a nonexistent # ghcr.io//wolfssl-ci-debs. IMG="ghcr.io/wolfssl/wolfssl-ci-debs:${{ inputs.ghcr-debs-tag }}" - if ! docker pull -q "$IMG" >/dev/null 2>&1; then + if ! timeout -k 10 300 docker pull -q "$IMG" >/dev/null 2>&1; then echo "::notice::ghcr bundle $IMG unavailable; using apt" exit 0 fi @@ -85,9 +88,25 @@ runs: NO_REC="--no-install-recommends" fi + # A wedged mirror hangs apt rather than failing it, so the retry loop + # below never fired and the job burned its whole budget instead. + # Defend in depth: apt drops a stalled connection after 30s and retries + # it (Acquire timeouts - this is what actually detects a wedge, in + # ~90s), `timeout` hard-kills an apt-get that wedged outside its own + # I/O loop, then the loop re-runs - re-reading apt-mirrors.txt, so a + # retry can land on a different mirror. The timeouts stay tight so + # every attempt fits the caller's timeout-minutes (as low as 4 min); + # apt resumes from archives/partial/, so a killed transfer is not + # restarted from scratch. + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30) + for i in $(seq 1 $RETRIES); do - if sudo apt-get update -q && \ - sudo apt-get install -y $NO_REC ${{ inputs.packages }}; then + # A previous attempt killed mid-unpack leaves dpkg needing this. + sudo dpkg --configure -a >/dev/null 2>&1 || true + if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \ + sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \ + $NO_REC ${{ inputs.packages }}; then exit 0 fi if [ "$i" -eq "$RETRIES" ]; then diff --git a/.github/ci-deps/packages-ubuntu-22.04-minimal.txt b/.github/ci-deps/packages-ubuntu-22.04-minimal.txt index c32e3ccb9cf..6fdbf8cfbba 100644 --- a/.github/ci-deps/packages-ubuntu-22.04-minimal.txt +++ b/.github/ci-deps/packages-ubuntu-22.04-minimal.txt @@ -4,6 +4,7 @@ autoconf automake build-essential +ccache crossbuild-essential-arm64 crossbuild-essential-armel crossbuild-essential-armhf diff --git a/.github/scripts/zephyr-4.x/zephyr-test.sh b/.github/scripts/zephyr-4.x/zephyr-test.sh index c225277cacc..66a2930d64c 100755 --- a/.github/scripts/zephyr-4.x/zephyr-test.sh +++ b/.github/scripts/zephyr-4.x/zephyr-test.sh @@ -201,7 +201,13 @@ echo "==> [container] Exporting Zephyr..." west zephyr-export echo "==> [container] Installing host packages (newlib, python3-venv)..." -sudo apt-get update -qq && sudo apt-get install -y -qq python3-venv libnewlib-dev >/dev/null 2>&1 || true +# `|| true` keeps this best-effort, but without a timeout a wedged mirror +# stalls here silently until the job budget runs out. +APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30) +sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -qq >/dev/null 2>&1 \ + && sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y -qq \ + python3-venv libnewlib-dev >/dev/null 2>&1 \ + || echo "==> [container] host package install skipped (apt unavailable)" python3 -m venv .venv source .venv/bin/activate pip3 install west diff --git a/.github/workflows/ci-deps-image.yml b/.github/workflows/ci-deps-image.yml index d91ac4fb877..76f1a65489d 100644 --- a/.github/workflows/ci-deps-image.yml +++ b/.github/workflows/ci-deps-image.yml @@ -183,21 +183,27 @@ jobs: set -euo pipefail K="${{ steps.check.outputs.kernel }}" # linuxkm.yml installs only the headers; the membrowse linuxkm targets - # also need the build toolchain. Bundle the union - each consumer - # installs its own subset offline. - PKGS=(build-essential autoconf automake libtool "linux-headers-$K") + # also need the build toolchain, and ccache-setup installs ccache + # offline from whatever this bundle staged. Bundle the union - each + # consumer installs its own subset offline. + PKGS=(build-essential autoconf automake libtool ccache + "linux-headers-$K") echo "Packages: ${PKGS[*]}" export DEBIAN_FRONTEND=noninteractive rm -rf debs && mkdir -p debs sudo apt-get clean - retry() { local i; for i in 1 2 3 4 5; do "$@" && return 0; sleep $((2**i)); done; "$@"; } - retry sudo apt-get update -q + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30) + # 2 attempts, not 5: this job's timeout-minutes is 20, and an + # attempt cut off mid-flight reports nothing. + retry() { local i; for i in 1 2; do "$@" && return 0; sleep 5; done; "$@"; } + retry sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q # The whole set is required and this bundle is small, so resolve it as # one closure and let any download failure fail the job. We push only # on success, so a transient mirror error keeps the last good bundle # rather than publishing a partial one - which the kernel-label skip # would then pin in place until the kernel next changes (~monthly). - retry sudo apt-get install -y --download-only "${PKGS[@]}" + retry sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \ + --download-only "${PKGS[@]}" sudo cp /var/cache/apt/archives/*.deb debs/ 2>/dev/null || true echo "Bundled $(ls debs/*.deb 2>/dev/null | wc -l) .deb files" test -n "$(ls debs/*.deb 2>/dev/null)" # headers are never preinstalled diff --git a/.github/workflows/cross-library.yml b/.github/workflows/cross-library.yml index af25dd52cf1..34bb4da8c41 100644 --- a/.github/workflows/cross-library.yml +++ b/.github/workflows/cross-library.yml @@ -64,10 +64,28 @@ jobs: run: | set -eux export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y --no-install-recommends \ - build-essential autoconf automake libtool pkg-config \ - git ca-certificates ${{ inputs.apt_packages }} + # These containers are bare images with no bash, so this step runs + # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it + # word-splits. + # A wedged mirror hangs apt instead of failing it. Acquire timeouts + # drop a stalled connection (and are what actually detects a wedge), + # `timeout` hard-kills apt-get if it wedges outside its own I/O loop, + # and the loop then retries - re-reading the mirror list. Two + # attempts at 60s+300s fit inside this job's timeout-minutes; apt + # resumes from archives/partial/, so a killed transfer is not lost. + APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30" + for i in 1 2; do + if timeout -k 10 60 apt-get $APT_OPTS update -q && \ + timeout -k 10 300 apt-get $APT_OPTS install -y \ + --no-install-recommends \ + build-essential autoconf automake libtool pkg-config \ + git ca-certificates ${{ inputs.apt_packages }}; then + break + fi + test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; } + echo "::warning::apt-get failed (attempt $i/2)" + sleep 5 + done # Building only needs the commit under test, not history. The break check # that needs history runs in the compile job, not here. @@ -129,10 +147,28 @@ jobs: run: | set -eux export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y --no-install-recommends \ - build-essential autoconf automake libtool pkg-config \ - git ca-certificates ${{ inputs.apt_packages }} + # These containers are bare images with no bash, so this step runs + # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it + # word-splits. + # A wedged mirror hangs apt instead of failing it. Acquire timeouts + # drop a stalled connection (and are what actually detects a wedge), + # `timeout` hard-kills apt-get if it wedges outside its own I/O loop, + # and the loop then retries - re-reading the mirror list. Two + # attempts at 60s+300s fit inside this job's timeout-minutes; apt + # resumes from archives/partial/, so a killed transfer is not lost. + APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30" + for i in 1 2; do + if timeout -k 10 60 apt-get $APT_OPTS update -q && \ + timeout -k 10 300 apt-get $APT_OPTS install -y \ + --no-install-recommends \ + build-essential autoconf automake libtool pkg-config \ + git ca-certificates ${{ inputs.apt_packages }}; then + break + fi + test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; } + echo "::warning::apt-get failed (attempt $i/2)" + sleep 5 + done # This job does not build wolfSSL, but the latest leg still checks out # wolfSSL history because check-break.sh scans commit messages here. The diff --git a/.github/workflows/falcon-interop.yml b/.github/workflows/falcon-interop.yml index ad6bc8f0137..1cf613e2ff4 100644 --- a/.github/workflows/falcon-interop.yml +++ b/.github/workflows/falcon-interop.yml @@ -91,8 +91,26 @@ jobs: steps: - name: Install build tools run: | - sudo apt-get update - sudo apt-get install -y ninja-build + # A wedged mirror hangs apt instead of failing it. Acquire timeouts + # drop a stalled connection, `timeout` hard-kills apt-get if it + # wedges anyway, and the loop then retries against a fresh mirror. + # Two attempts at 60s+300s stay inside this job's timeout-minutes. + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30) + apt_retry() { + local i + for i in 1 2; do + if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \ + sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then + return 0 + fi + echo "::warning::apt-get failed (attempt $i/2)" + sleep 5 + done + echo "::error::apt-get failed after 2 attempts" + return 1 + } + apt_retry ninja-build # Check out wolfSSL first: actions/checkout runs "git clean -ffdx", which # would delete an untracked oqs-install/ placed in the workspace by the @@ -161,8 +179,26 @@ jobs: steps: - name: Install build tools run: | - sudo apt-get update - sudo apt-get install -y autoconf automake libtool + # A wedged mirror hangs apt instead of failing it. Acquire timeouts + # drop a stalled connection, `timeout` hard-kills apt-get if it + # wedges anyway, and the loop then retries against a fresh mirror. + # Two attempts at 60s+300s stay inside this job's timeout-minutes. + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30) + apt_retry() { + local i + for i in 1 2; do + if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \ + sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then + return 0 + fi + echo "::warning::apt-get failed (attempt $i/2)" + sleep 5 + done + echo "::error::apt-get failed after 2 attempts" + return 1 + } + apt_retry autoconf automake libtool - name: Checkout wolfSSL uses: actions/checkout@v5 diff --git a/.github/workflows/sbom.yml b/.github/workflows/sbom.yml index 4e2b3fcd950..8a6389a6c91 100644 --- a/.github/workflows/sbom.yml +++ b/.github/workflows/sbom.yml @@ -822,7 +822,25 @@ jobs: - name: Install build deps + SBOM validators run: | - sudo apt-get update + # A wedged mirror hangs apt instead of failing it. Acquire timeouts + # drop a stalled connection, `timeout` hard-kills apt-get if it + # wedges anyway, and the loop then retries against a fresh mirror. + # Two attempts at 60s+300s stay inside this job's timeout-minutes. + APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30) + apt_retry() { + local i + for i in 1 2; do + if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \ + sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then + return 0 + fi + echo "::warning::apt-get failed (attempt $i/2)" + sleep 5 + done + echo "::error::apt-get failed after 2 attempts" + return 1 + } # bison + autotools-dev are required by strace's ./bootstrap. # gcc-multilib + g++-multilib give strace's --enable-mpers=check # the 32-bit/x32 compilers it needs - without them mpers is @@ -830,7 +848,7 @@ jobs: # syscalls, diverging from what bomsh's devcontainer produces. # The rest mirror bomsh's .devcontainer/Dockerfile bomtrace3 # stage. - sudo apt-get install -y build-essential autoconf automake libtool \ + apt_retry build-essential autoconf automake libtool \ bison autotools-dev gcc-multilib g++-multilib \ python3 python3-pip git python3 -m pip install --user --upgrade pip